diff --git a/src/datatrove/pipeline/readers/base.py b/src/datatrove/pipeline/readers/base.py index e7f4569..487cad5 100644 --- a/src/datatrove/pipeline/readers/base.py +++ b/src/datatrove/pipeline/readers/base.py @@ -62,7 +62,7 @@ class BaseReader(PipelineStep): "text": data.pop(self.text_key, ""), "id": data.pop(self.id_key, f"{path}/{id_in_file}"), "media": data.pop("media", []), - "metadata": data.pop("metadata", {}) | data, # remaining data goes into metadata + # "metadata": data.pop("metadata", {}) | data, # remaining data goes into metadata } def get_document_from_dict(self, data: dict, source_file: str, id_in_file: int | str): diff --git a/src/datatrove/utils/tokenization.py b/src/datatrove/utils/tokenization.py index 5ce298f..3ff067c 100644 --- a/src/datatrove/utils/tokenization.py +++ b/src/datatrove/utils/tokenization.py @@ -48,12 +48,13 @@ class PipelineStepWithTokenizer(PipelineStep, ABC): if not self.tokenizer_name_or_path: raise ValueError("self.tokenizer_name_or_path needs to be set!") self._tokenizer = load_tokenizer(self.tokenizer_name_or_path) - if self._post_processor: - self._tokenizer.post_processor = self._post_processor - elif self.eos_token: - self._tokenizer.post_processor = TemplateProcessing( - single="$A ", - special_tokens=[("", self.tokenizer.token_to_id(self.eos_token))], - pair=None, - ) + self.eos_token = self._tokenizer.token_to_id(self.eos_token) + # if self._post_processor: + # self._tokenizer.post_processor = self._post_processor + # elif self.eos_token: + # self._tokenizer.post_processor = TemplateProcessing( + # single="$A ", + # special_tokens=[("", self.tokenizer.token_to_id(self.eos_token))], + # pair=None, + # ) return self._tokenizer