diff --git a/src/datatrove/pipeline/readers/base.py b/src/datatrove/pipeline/readers/base.py index e7f4569..487cad5 100644 --- a/src/datatrove/pipeline/readers/base.py +++ b/src/datatrove/pipeline/readers/base.py @@ -62,7 +62,7 @@ class BaseReader(PipelineStep): "text": data.pop(self.text_key, ""), "id": data.pop(self.id_key, f"{path}/{id_in_file}"), "media": data.pop("media", []), - "metadata": data.pop("metadata", {}) | data, # remaining data goes into metadata + # "metadata": data.pop("metadata", {}) | data, # remaining data goes into metadata } def get_document_from_dict(self, data: dict, source_file: str, id_in_file: int | str): diff --git a/src/datatrove/pipeline/tokens/tokenizer.py b/src/datatrove/pipeline/tokens/tokenizer.py index d6040a0..e86f9bb 100644 --- a/src/datatrove/pipeline/tokens/tokenizer.py +++ b/src/datatrove/pipeline/tokens/tokenizer.py @@ -358,7 +358,7 @@ class DocumentTokenizer(PipelineStepWithTokenizer): # tokenize document's text in batches to go faster – we compute loss values independently if needed for batch in batched(data, self.batch_size): with self.track_time(unit="batch"): - encoded_batch: list[Encoding] = self.tokenizer.encode_batch([document.text for document in batch]) + encoded_batch: list[Encoding] = self.tokenizer.encode_batch([document.text[0] for document in batch]) for document, encoded in zip(batch, encoded_batch): tokens = encoded.ids loss_values = self.get_loss_values(document, encoded) diff --git a/src/datatrove/utils/tokenization.py b/src/datatrove/utils/tokenization.py index 5ce298f..3ff067c 100644 --- a/src/datatrove/utils/tokenization.py +++ b/src/datatrove/utils/tokenization.py @@ -48,12 +48,13 @@ class PipelineStepWithTokenizer(PipelineStep, ABC): if not self.tokenizer_name_or_path: raise ValueError("self.tokenizer_name_or_path needs to be set!") self._tokenizer = load_tokenizer(self.tokenizer_name_or_path) - if self._post_processor: - self._tokenizer.post_processor = self._post_processor - elif self.eos_token: - self._tokenizer.post_processor = TemplateProcessing( - single="$A ", - special_tokens=[("", self.tokenizer.token_to_id(self.eos_token))], - pair=None, - ) + self.eos_token = self._tokenizer.token_to_id(self.eos_token) + # if self._post_processor: + # self._tokenizer.post_processor = self._post_processor + # elif self.eos_token: + # self._tokenizer.post_processor = TemplateProcessing( + # single="$A ", + # special_tokens=[("", self.tokenizer.token_to_id(self.eos_token))], + # pair=None, + # ) return self._tokenizer