Set up the Open Source build for MediaPipe Tasks Web
PiperOrigin-RevId: 486181061
This commit is contained in:
Sebastian Schmidt
committed by
Copybara-Service
Copybara-Service
parent
5fd3701cfd
commit
f05868d8e0
@@ -0,0 +1,21 @@
|
||||
# This package contains options shared by all MediaPipe Tasks for Web.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
|
||||
|
||||
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "category",
|
||||
srcs = ["category.d.ts"],
|
||||
)
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "classifications",
|
||||
srcs = ["classifications.d.ts"],
|
||||
deps = [":category"],
|
||||
)
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "landmark",
|
||||
srcs = ["landmark.d.ts"],
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/** A classification category. */
|
||||
export interface Category {
|
||||
/** The probability score of this label category. */
|
||||
score: number;
|
||||
|
||||
/** The index of the category in the corresponding label file. */
|
||||
index: number;
|
||||
|
||||
/**
|
||||
* The label of this category object. Defaults to an empty string if there is
|
||||
* no category.
|
||||
*/
|
||||
categoryName: string;
|
||||
|
||||
/**
|
||||
* The display name of the label, which may be translated for different
|
||||
* locales. For example, a label, "apple", may be translated into Spanish for
|
||||
* display purpose, so that the `display_name` is "manzana". Defaults to an
|
||||
* empty string if there is no display name.
|
||||
*/
|
||||
displayName: string;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {Category} from '../../../../tasks/web/components/containers/category';
|
||||
|
||||
/** List of predicted categories with an optional timestamp. */
|
||||
export interface ClassificationEntry {
|
||||
/**
|
||||
* The array of predicted categories, usually sorted by descending scores,
|
||||
* e.g., from high to low probability.
|
||||
*/
|
||||
categories: Category[];
|
||||
|
||||
/**
|
||||
* The optional timestamp (in milliseconds) associated to the classification
|
||||
* entry. This is useful for time series use cases, e.g., audio
|
||||
* classification.
|
||||
*/
|
||||
timestampMs?: number;
|
||||
}
|
||||
|
||||
/** Classifications for a given classifier head. */
|
||||
export interface Classifications {
|
||||
/** A list of classification entries. */
|
||||
entries: ClassificationEntry[];
|
||||
|
||||
/**
|
||||
* The index of the classifier head these categories refer to. This is
|
||||
* useful for multi-head models.
|
||||
*/
|
||||
headIndex: number;
|
||||
|
||||
/**
|
||||
* The name of the classifier head, which is the corresponding tensor
|
||||
* metadata name.
|
||||
*/
|
||||
headName: string;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Landmark represents a point in 3D space with x, y, z coordinates. If
|
||||
* normalized is true, the landmark coordinates is normalized respect to the
|
||||
* dimension of image, and the coordinates values are in the range of [0,1].
|
||||
* Otherwise, it represenet a point in world coordinates.
|
||||
*/
|
||||
export class Landmark {
|
||||
/** The x coordinates of the landmark. */
|
||||
x: number;
|
||||
|
||||
/** The y coordinates of the landmark. */
|
||||
y: number;
|
||||
|
||||
/** The z coordinates of the landmark. */
|
||||
z: number;
|
||||
|
||||
/** Whether this landmark is normalized with respect to the image size. */
|
||||
normalized: boolean;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
# This package contains options shared by all MediaPipe Tasks for Web.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
|
||||
|
||||
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "classifier_options",
|
||||
srcs = ["classifier_options.ts"],
|
||||
deps = [
|
||||
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_jspb_proto",
|
||||
"//mediapipe/tasks/web/core:classifier_options",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "classifier_result",
|
||||
srcs = ["classifier_result.ts"],
|
||||
deps = [
|
||||
"//mediapipe/tasks/cc/components/containers/proto:classifications_jspb_proto",
|
||||
"//mediapipe/tasks/web/components/containers:classifications",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "base_options",
|
||||
srcs = ["base_options.ts"],
|
||||
deps = [
|
||||
"//mediapipe/tasks/cc/core/proto:base_options_jspb_proto",
|
||||
"//mediapipe/tasks/cc/core/proto:external_file_jspb_proto",
|
||||
"//mediapipe/tasks/web/core",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/base_options_pb';
|
||||
import {ExternalFile} from '../../../../tasks/cc/core/proto/external_file_pb';
|
||||
import {BaseOptions} from '../../../../tasks/web/core/base_options';
|
||||
|
||||
// The OSS JS API does not support the builder pattern.
|
||||
// tslint:disable:jspb-use-builder-pattern
|
||||
|
||||
/**
|
||||
* Converts a BaseOptions API object to its Protobuf representation.
|
||||
* @throws If neither a model assset path or buffer is provided
|
||||
*/
|
||||
export async function convertBaseOptionsToProto(baseOptions: BaseOptions):
|
||||
Promise<BaseOptionsProto> {
|
||||
if (baseOptions.modelAssetPath && baseOptions.modelAssetBuffer) {
|
||||
throw new Error(
|
||||
'Cannot set both baseOptions.modelAssetPath and baseOptions.modelAssetBuffer');
|
||||
}
|
||||
if (!baseOptions.modelAssetPath && !baseOptions.modelAssetBuffer) {
|
||||
throw new Error(
|
||||
'Either baseOptions.modelAssetPath or baseOptions.modelAssetBuffer must be set');
|
||||
}
|
||||
|
||||
let modelAssetBuffer = baseOptions.modelAssetBuffer;
|
||||
if (!modelAssetBuffer) {
|
||||
const response = await fetch(baseOptions.modelAssetPath!.toString());
|
||||
modelAssetBuffer = new Uint8Array(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
const proto = new BaseOptionsProto();
|
||||
const externalFile = new ExternalFile();
|
||||
externalFile.setFileContent(modelAssetBuffer);
|
||||
proto.setModelAsset(externalFile);
|
||||
return proto;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {ClassifierOptions as ClassifierOptionsProto} from '../../../../tasks/cc/components/processors/proto/classifier_options_pb';
|
||||
import {ClassifierOptions} from '../../../../tasks/web/core/classifier_options';
|
||||
|
||||
/**
|
||||
* Converts a ClassifierOptions object to its Proto representation, optionally
|
||||
* based on existing definition.
|
||||
* @param options The options object to convert to a Proto. Only options that
|
||||
* are expliclty provided are set.
|
||||
* @param baseOptions A base object that options can be merged into.
|
||||
*/
|
||||
export function convertClassifierOptionsToProto(
|
||||
options: ClassifierOptions,
|
||||
baseOptions?: ClassifierOptionsProto): ClassifierOptionsProto {
|
||||
const classifierOptions =
|
||||
baseOptions ? baseOptions.clone() : new ClassifierOptionsProto();
|
||||
if (options.displayNamesLocale) {
|
||||
classifierOptions.setDisplayNamesLocale(options.displayNamesLocale);
|
||||
} else if (options.displayNamesLocale === undefined) {
|
||||
classifierOptions.clearDisplayNamesLocale();
|
||||
}
|
||||
|
||||
if (options.maxResults) {
|
||||
classifierOptions.setMaxResults(options.maxResults);
|
||||
} else if ('maxResults' in options) { // Check for undefined
|
||||
classifierOptions.clearMaxResults();
|
||||
}
|
||||
|
||||
if (options.scoreThreshold) {
|
||||
classifierOptions.setScoreThreshold(options.scoreThreshold);
|
||||
} else if ('scoreThreshold' in options) { // Check for undefined
|
||||
classifierOptions.clearScoreThreshold();
|
||||
}
|
||||
|
||||
if (options.categoryAllowlist) {
|
||||
classifierOptions.setCategoryAllowlistList(options.categoryAllowlist);
|
||||
} else if ('categoryAllowlist' in options) { // Check for undefined
|
||||
classifierOptions.clearCategoryAllowlistList();
|
||||
}
|
||||
|
||||
if (options.categoryDenylist) {
|
||||
classifierOptions.setCategoryDenylistList(options.categoryDenylist);
|
||||
} else if ('categoryDenylist' in options) { // Check for undefined
|
||||
classifierOptions.clearCategoryDenylistList();
|
||||
}
|
||||
return classifierOptions;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {ClassificationEntry as ClassificationEntryProto, ClassificationResult} from '../../../../tasks/cc/components/containers/proto/classifications_pb';
|
||||
import {ClassificationEntry, Classifications} from '../../../../tasks/web/components/containers/classifications';
|
||||
|
||||
const DEFAULT_INDEX = -1;
|
||||
const DEFAULT_SCORE = 0.0;
|
||||
|
||||
/**
|
||||
* Converts a ClassificationEntry proto to the ClassificationEntry result
|
||||
* type.
|
||||
*/
|
||||
function convertFromClassificationEntryProto(source: ClassificationEntryProto):
|
||||
ClassificationEntry {
|
||||
const categories = source.getCategoriesList().map(category => {
|
||||
return {
|
||||
index: category.getIndex() ?? DEFAULT_INDEX,
|
||||
score: category.getScore() ?? DEFAULT_SCORE,
|
||||
displayName: category.getDisplayName() ?? '',
|
||||
categoryName: category.getCategoryName() ?? '',
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
categories,
|
||||
timestampMs: source.getTimestampMs(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a ClassificationResult proto to a list of classifications.
|
||||
*/
|
||||
export function convertFromClassificationResultProto(
|
||||
classificationResult: ClassificationResult) : Classifications[] {
|
||||
const result: Classifications[] = [];
|
||||
for (const classificationsProto of
|
||||
classificationResult.getClassificationsList()) {
|
||||
const classifications: Classifications = {
|
||||
entries: classificationsProto.getEntriesList().map(
|
||||
entry => convertFromClassificationEntryProto(entry)),
|
||||
headIndex: classificationsProto.getHeadIndex() ?? DEFAULT_INDEX,
|
||||
headName: classificationsProto.getHeadName() ?? '',
|
||||
};
|
||||
result.push(classifications);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# This package contains options shared by all MediaPipe Tasks for Web.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
|
||||
|
||||
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "core",
|
||||
srcs = [
|
||||
"base_options.d.ts",
|
||||
"wasm_loader_options.d.ts",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "classifier_options",
|
||||
srcs = [
|
||||
"classifier_options.d.ts",
|
||||
],
|
||||
deps = [":core"],
|
||||
)
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Placeholder for internal dependency on trusted resource url
|
||||
|
||||
/** Options to configure MediaPipe Tasks in general. */
|
||||
export interface BaseOptions {
|
||||
/**
|
||||
* The model path to the model asset file. Only one of `modelAssetPath` or
|
||||
* `modelAssetBuffer` can be set.
|
||||
*/
|
||||
modelAssetPath?: string;
|
||||
/**
|
||||
* A buffer containing the model aaset. Only one of `modelAssetPath` or
|
||||
* `modelAssetBuffer` can be set.
|
||||
*/
|
||||
modelAssetBuffer?: Uint8Array;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {BaseOptions} from '../../../tasks/web/core/base_options';
|
||||
|
||||
/** Options to configure the Mediapipe Classifier Task. */
|
||||
export interface ClassifierOptions {
|
||||
/** Options to configure the loading of the model assets. */
|
||||
baseOptions?: BaseOptions;
|
||||
|
||||
/**
|
||||
* The locale to use for display names specified through the TFLite Model
|
||||
* Metadata, if any. Defaults to English.
|
||||
*/
|
||||
displayNamesLocale?: string|undefined;
|
||||
|
||||
/** The maximum number of top-scored detection results to return. */
|
||||
maxResults?: number|undefined;
|
||||
|
||||
/**
|
||||
* Overrides the value provided in the model metadata. Results below this
|
||||
* value are rejected.
|
||||
*/
|
||||
scoreThreshold?: number|undefined;
|
||||
|
||||
/**
|
||||
* Allowlist of category names. If non-empty, detection results whose category
|
||||
* name is not in this set will be filtered out. Duplicate or unknown category
|
||||
* names are ignored. Mutually exclusive with `categoryDenylist`.
|
||||
*/
|
||||
categoryAllowlist?: string[]|undefined;
|
||||
|
||||
/**
|
||||
* Denylist of category names. If non-empty, detection results whose category
|
||||
* name is in this set will be filtered out. Duplicate or unknown category
|
||||
* names are ignored. Mutually exclusive with `categoryAllowlist`.
|
||||
*/
|
||||
categoryDenylist?: string[]|undefined;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Placeholder for internal dependency on trusted resource url
|
||||
|
||||
/** An object containing the locations of all Wasm assets */
|
||||
export interface WasmLoaderOptions {
|
||||
/** The path to the Wasm loader script. */
|
||||
wasmLoaderPath: string;
|
||||
/** The path to the Wasm binary. */
|
||||
wasmBinaryPath: string;
|
||||
}
|
||||
Reference in New Issue
Block a user