48 lines
1.5 KiB
Protocol Buffer
48 lines
1.5 KiB
Protocol Buffer
// Copyright (C) 2017 The Android Open Source Project
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
syntax = "proto2";
|
|
option optimize_for = LITE_RUNTIME;
|
|
|
|
package libtextclassifier;
|
|
|
|
// Represents a codepoint range [start, end) with its role for tokenization.
|
|
message TokenizationCodepointRange {
|
|
optional int32 start = 1;
|
|
optional int32 end = 2;
|
|
|
|
// Role of the codepoints in the range.
|
|
enum Role {
|
|
// Concatenates the codepoint to the current run of codepoints.
|
|
DEFAULT_ROLE = 0;
|
|
|
|
// Splits a run of codepoints before the current codepoint.
|
|
SPLIT_BEFORE = 0x1;
|
|
|
|
// Splits a run of codepoints after the current codepoint.
|
|
SPLIT_AFTER = 0x2;
|
|
|
|
// Discards the codepoint.
|
|
DISCARD_CODEPOINT = 0x4;
|
|
|
|
// Common values:
|
|
// Splits on the characters and discards them. Good e.g. for the space
|
|
// character.
|
|
WHITESPACE_SEPARATOR = 0x7;
|
|
// Each codepoint will be a separate token. Good e.g. for Chinese
|
|
// characters.
|
|
TOKEN_SEPARATOR = 0x3;
|
|
}
|
|
optional Role role = 3;
|
|
}
|