Project import generated by Copybara.
GitOrigin-RevId: 1e13be30e2c6838d4a2ff768a39c414bc80534bb
This commit is contained in:
committed by
Sebastian Schmidt
parent
63e679d99c
commit
4dc4b19ddb
+38
@@ -0,0 +1,38 @@
|
||||
This diff patches commit a0f49e63, which has been released os part of Tensorflow
|
||||
Text 2.9. We can remove this patch once we update our dependency. This has not
|
||||
been done yet due to a Protobuf incompatibility introduced by this update.
|
||||
|
||||
diff --git a/tensorflow_text/core/kernels/regex_split.cc b/tensorflow_text/core/kernels/regex_split.cc
|
||||
index 473b922e..db422781 100644
|
||||
--- a/tensorflow_text/core/kernels/regex_split.cc
|
||||
+++ b/tensorflow_text/core/kernels/regex_split.cc
|
||||
@@ -45,24 +45,24 @@ void RegexSplitImpl(absl::string_view input, const RE2& re2,
|
||||
tokens->push_back(token);
|
||||
// Mark the end of the last token
|
||||
begin_offsets->push_back(token.data() - input.data());
|
||||
- end_offsets->push_back(token.data() + token.length() - input.begin());
|
||||
+ end_offsets->push_back(token.data() + token.length() - input.data());
|
||||
}
|
||||
|
||||
if (should_include_delim) {
|
||||
// If desired, include the deliminator as a token.
|
||||
tokens->push_back(extracted_delim_token);
|
||||
// Mark the end of the token at the end of the beginning of the delimiter.
|
||||
- begin_offsets->push_back(extracted_delim_token.data() - input.begin());
|
||||
+ begin_offsets->push_back(extracted_delim_token.data() - input.data());
|
||||
end_offsets->push_back(extracted_delim_token.data() +
|
||||
- extracted_delim_token.length() - input.begin());
|
||||
+ extracted_delim_token.length() - input.data());
|
||||
}
|
||||
}
|
||||
|
||||
// Close the last token.
|
||||
if (!leftover.empty()) {
|
||||
tokens->push_back(leftover);
|
||||
- begin_offsets->push_back(leftover.data() - input.begin());
|
||||
- end_offsets->push_back(leftover.data() + leftover.length() - input.begin());
|
||||
+ begin_offsets->push_back(leftover.data() - input.data());
|
||||
+ end_offsets->push_back(leftover.data() + leftover.length() - input.data());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user