| ... |
|
01_download_warc.py
|
3.72 KB |
|
02_bis_extract_html_get_image_urls_new_rules.py
|
6.87 KB |
|
02_extract_html_get_image_urls.py
|
5.76 KB |
|
02_parallel_extract_html_get_image_urls.py
|
1.59 KB |
|
03_dl_images_create_dataset.py
|
6.5 KB |
|
03_parallel_dl_images_create_dataset.py
|
1.05 KB |
|
04_merge_web_docs_with_images.py
|
7.44 KB |
|
05_filtering_web_docs.py
|
12.81 KB |
|
06_01_create_set_image_urls_in_webdocs.py
|
4 KB |
|
06_02_merge_sets_image_urls_in_webdocs.py
|
2.17 KB |
|
06_03_remove_image_duplicates.py
|
5.64 KB |
|
07_01_nsfw_image_filtering.py
|
8.4 KB |
|
07_02_nsfw_image_visualization.py
|
3.16 KB |
|
07_03_nsfw_image_removal.py
|
3.83 KB |
|
08_01_prepare_urldedup.py
|
3.82 KB |
|
08_02_urldedup.py
|
3.99 KB |
|
09_01_create_web_docs_texts_only.py
|
2.31 KB |
|
09_02_get_domain_to_positions.py
|
2.95 KB |
|
09_03_split_domain_to_positions.py
|
1.65 KB |
|
09_04_get_domain_to_duplicated_texts.py
|
4.34 KB |
|
09_05_merge_domain_to_duplicated_texts_sharded.py
|
4.48 KB |
|
09_06_line_dedup.py
|
4.81 KB |
|
09_07_merge_web_docs_texts_only_and_rest.py
|
3.13 KB |
|
10_final_cleaning.py
|
5.22 KB |
|
11_01_create_set_img_urls.py
|
2.55 KB |
|
11_02_get_docs_to_remove_by_set_img_urls_dedup.py
|
4.14 KB |
|
11_03_set_img_urls_dedup.py
|
5.05 KB |
|
12_01_find_opt_out_images.py
|
3.47 KB |
|
12_02_remove_opt_out_images.py
|
3.97 KB |
|
13_final_processing.py
|
11.87 KB |
|
14_01_filter_perplexity_with_language_model.py
|
6.16 KB |
|
15_01_find_urls_obelics.py
|
2.96 KB |
|
15_02_find_opt_out.py
|
3.76 KB |
|
15_03_remove_opt_out_documents.py
|
3.68 KB |
|
15_04_remove_opt_out_images.py
|
4.25 KB |