Merge pull request #56 from chengzheng345/main

Remove the hosts Settings for multiple machines and only support sing…

Author: anxiangsir

examples/llava_ov_1_5/sample_packing/1_s1_get_tokenlens_v3-sft.py

@@ -16,29 +16,12 @@
 import multiprocessing
 from multiprocessing import Pool, Manager, Value
 from tqdm import tqdm
-from tool import get_ip_info,cfg
+from tool import cfg,get_init_file
 
 # Declares a global cross-process counter (defined in the main module for child processes to inherit)
 global_total_counter = None
-MAX_TOKEN_LEN = cfg['sample']['max_len']
 task_type = cfg['sample']['task_type']
-DEFAULT_DIRECTORY = Path(cfg['data']['directory'])
-try:
-    ip_index,_,_=get_ip_info(cfg['hosts'])
-    print('success init ip ,>>>>>>>>>>>>>>>>>>>>>>>>')
-except:
-    print(f"getting ip_index error, default to 0")
-    ip_index=0
-DEFAULT_DIRECTORY=Path(os.path.join(DEFAULT_DIRECTORY,f'part_{ip_index:02d}'))
-save_files_dir=os.path.join(DEFAULT_DIRECTORY,"save_files")
-if os.path.exists(save_files_dir) is False:
-    os.makedirs(save_files_dir)
-
-OUTPUT_FILE = Path(cfg['data']['output_base'])
-TOKEN_INFO_FILE = Path(cfg['data']['output_token'])
-OUTPUT_FILE=os.path.join(save_files_dir,OUTPUT_FILE)
-TOKEN_INFO_FILE=os.path.join(save_files_dir,TOKEN_INFO_FILE)
-
+TOKEN_INFO_FILE,OUTPUT_FILE,MAX_TOKEN_LEN,save_files_dir,big_dir,DEFAULT_DIRECTORY=get_init_file()
 
 CKPT_DIR = cfg['model']['checkpoint']
 MIN_PIXELS = cfg['image']['min_pixels']

examples/llava_ov_1_5/sample_packing/2_do_hashbacket.py

@@ -2,7 +2,6 @@
 from pprint import pprint
 import os
 import yaml
-from tool import get_ip_info
 import random
 from tool import get_init_file
 
@@ -26,7 +25,7 @@ def get_hs(hs):
     return mean,min_,max_,num
 
 def init():
-    input_file ,MAX_TOKEN_LEN,save_files_dir,_,_= get_init_file()
+    input_file ,_,MAX_TOKEN_LEN,save_files_dir,_,_= get_init_file()
     if not os.path.exists(input_file):
         print(f" file {input_file} does not exist!" )
         processor=None

examples/llava_ov_1_5/sample_packing/3_s2_prepare_rawsamples-vqa.py

@@ -7,7 +7,7 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from tool import get_init_file
 
-input_token_file ,MAX_TOKEN_LEN,save_files_dir,big_dir,DEFAULT_DIRECTORY= get_init_file()
+input_token_file,_ ,MAX_TOKEN_LEN,save_files_dir,big_dir,DEFAULT_DIRECTORY= get_init_file()
 SRC_DIR_IMGS = DEFAULT_DIRECTORY   # The storage location of image data
 SRC_DIR_JSONS = DEFAULT_DIRECTORY   # The storage location of json data
 SRC_DST_EXTENSIONS = ("jpg", "json")

examples/llava_ov_1_5/sample_packing/4_convert_packedsample_to_wds.py

@@ -157,7 +157,7 @@ def write_config(path: EPath, media=None, template_func=None, class_name=None):
 
 def _add_arguments(parser: argparse.ArgumentParser):
 
-    input_token_file ,MAX_TOKEN_LEN,save_files_dir,big_dir,DEFAULT_DIRECTORY= get_init_file()
+    input_token_file ,_,MAX_TOKEN_LEN,save_files_dir,big_dir,DEFAULT_DIRECTORY= get_init_file()
     output_dir=DEFAULT_DIRECTORY+'_wds'
     last_save_dir_json=os.path.join(save_files_dir,"row_packing_jsons")
     last_save_dir_image=os.path.join(save_files_dir,"row_packing_images")

examples/llava_ov_1_5/sample_packing/5_make_mix_wds_config.py

@@ -3,7 +3,6 @@
 Download the required data from [LLaVA-One-Vision-1.5-Mid-Training-85M](https://huggingface.co/datasets/lmms-lab/LLaVA-One-Vision-1.5-Mid-Training-85M/tree/main) 
 ## 2. Configure config.yaml
 The following are the key parameter configurations in the config.yaml file:
-- **hosts**: Specify the IP for offline packing. For a single machine, only one IP needs to be entered.
 - **hf_data**: The address of the downloaded data.
 - **directory**: The save address for processing results.
 - **checkpoint**: The address of the tokenizer used for VLM.
@@ -16,8 +15,6 @@ Run the offline_packing_pipeline.sh script. This script will sequentially execut
 3. Perform packing on the samples.
 4. Prepare for generating WebDataset.
 5. Package the packing results into WebDataset.
-6. Generate the configuration file for WebDataset.
-
 
 
 

examples/llava_ov_1_5/sample_packing/README.md

@@ -1,4 +1,3 @@
-hosts: "host_1.txt"
 hf_data: "your_LLaVA-One-Vision-1.5-Mid-Training-85M_data_path"
 data:
   directory: "your_to_save_result_path"

examples/llava_ov_1_5/sample_packing/config.yaml

@@ -1,6 +1,6 @@
 from datasets import load_dataset
 from multiprocessing import Pool
-from tool import cfg,get_ip_info,get_init_file
+from tool import cfg,get_init_file
 import os
 from functools import partial
 from tqdm import tqdm
@@ -36,11 +36,9 @@ def check_image(image_path) -> bool:
     except Exception as e:
         return False
 
-def parese_dataset(data_item,ip_indx,ip_num,dst_dir):
+def parse_dataset(data_item,dst_dir):
     try:
         index, item = data_item
-        if index%ip_num!=ip_indx:
-            return 
         name=item['id'].replace('/','_')
         name=os.path.splitext(name)[0]
 
@@ -76,12 +74,11 @@ def parese_dataset(data_item,ip_indx,ip_num,dst_dir):
 
 def main(workers):
     data_path=cfg['hf_data']
-    TOKEN_INFO_FILE,MAX_TOKEN_LEN,save_files_dir,big_dir,DEFAULT_DIRECTORY=get_init_file()
-    dataset = load_dataset(data_path,data_files='*/*.parquet', split="train", streaming=True) 
+    DEFAULT_DIRECTORY=get_init_file()[-1]
+    dataset = load_dataset(data_path,data_files='*/*/*.parquet', split="train", streaming=True) 
     data_iter = enumerate(dataset)
-    ip_indx,ip_num,_=get_ip_info()
-    with Pool(processes=workers) as pool, tqdm(total=8.5e8, desc="copy") as bar:
-        for _ in pool.imap_unordered(partial(parese_dataset,ip_indx=ip_indx,ip_num=ip_num,dst_dir=DEFAULT_DIRECTORY), data_iter):
+    with Pool(processes=workers) as pool, tqdm(total=8.5e8, desc="parsing data") as bar:
+        for _ in pool.imap_unordered(partial(parse_dataset,dst_dir=DEFAULT_DIRECTORY), data_iter):
             bar.update()
 
 if __name__=="__main__":

examples/llava_ov_1_5/sample_packing/huggingface_data_parse.py

@@ -17,7 +17,6 @@ docker exec -it "$CONTAINER_NAME" bash -c '
     run_python_script "2_do_hashbacket.py"
     run_python_script "3_s2_prepare_rawsamples-vqa.py"
     run_python_script "4_convert_packedsample_to_wds.py"
-    run_python_script "5_make_mix_wds_config.py"
 
     echo "─────────────────All processing workflows have been successfully completed.───────────────────"
 '

examples/llava_ov_1_5/sample_packing/offline_packing_pipeline.sh

@@ -5,52 +5,23 @@
 import pickle
 from tqdm import tqdm
 
-
 config='config.yaml'
 with open(config, 'r', encoding='utf-8') as f:
     cfg = yaml.safe_load(f)
 
-def get_ip_info(ip_file=cfg['hosts']):
-    hostname = socket.gethostname()
-    local_ip = socket.gethostbyname(hostname)
-    ips=get_ips(ip_file)
-    ip_indx=ips[local_ip]
-    ip_num=len(ips)
-    return ip_indx,ip_num,local_ip
-
-def get_ips(ip_file):
-    with open(ip_file, "r") as f:
-        ip_list = [line.strip() for line in f if line.strip()]
-        ip_list=sorted(ip_list)
-    res={ip:i for i,ip in enumerate(ip_list)}
-    return res
-
-def get_split_datas_by_ips(ip_file,datas):
-    ip_index,ip_num,_=get_ip_info(ip_file)
-    num_datas=len(datas)
-    step=num_datas//ip_num
-    return datas[ip_index*step:(ip_index+1)*step] if ip_index<ip_num-1 else datas[ip_index*step:num_datas]
-
 def get_init_file():
     MAX_TOKEN_LEN = cfg['sample']['max_len']
     big_dir = Path(cfg['data']['directory'])
-    try:
-        ip_index,_,_=get_ip_info(cfg['hosts'])
-        print('SUNCCESS ->>>>>>>>>>>>')
-    except:
-        print(f"getting ip_index error, default to 0")
-        ip_index=0
+    ip_index=0
     DEFAULT_DIRECTORY=os.path.join(big_dir,f'part_{ip_index:02d}')
-    print(DEFAULT_DIRECTORY)
     save_files_dir=os.path.join(DEFAULT_DIRECTORY,"save_files")
     os.makedirs(save_files_dir,exist_ok=True)
     OUTPUT_FILE = cfg['data']['output_base']
     TOKEN_INFO_FILE = cfg['data']['output_token']
-
     OUTPUT_FILE=os.path.join(save_files_dir,OUTPUT_FILE)
     TOKEN_INFO_FILE=os.path.join(save_files_dir,TOKEN_INFO_FILE)
 
-    return TOKEN_INFO_FILE,MAX_TOKEN_LEN,save_files_dir,big_dir,DEFAULT_DIRECTORY
+    return TOKEN_INFO_FILE,OUTPUT_FILE,MAX_TOKEN_LEN,save_files_dir,big_dir,DEFAULT_DIRECTORY
 
 def get_num_boxs():
     pairs_dir=cfg['data']['directory']
@@ -63,10 +34,4 @@ def get_num_boxs():
                 bin_boxes = pickle.load(f)
                 box_num+=len(bin_boxes)
                 sample_num+=sum([len(box) for box in bin_boxes])
-    return box_num,sample_num
-
-if __name__ == "__main__":
-    file_path=os.path.join('/vlm/chengzheng/datasets/pdf_datas/part_00/save_files','bins_boxs.pkl')
-    with open(file_path, 'rb') as f:
-        bin_boxes = pickle.load(f)
-        print(len(bin_boxes))
+    return box_num,sample_num