diff --git a/mediapipe/tasks/web/audio/audio_classifier/BUILD b/mediapipe/tasks/web/audio/audio_classifier/BUILD index c419d3b9..6f785dd0 100644 --- a/mediapipe/tasks/web/audio/audio_classifier/BUILD +++ b/mediapipe/tasks/web/audio/audio_classifier/BUILD @@ -25,7 +25,7 @@ mediapipe_ts_library( "//mediapipe/tasks/web/components/processors:classifier_result", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:classifier_options", - "//mediapipe/tasks/web/core:task_runner", + "//mediapipe/web/graph_runner:graph_runner_ts", ], ) @@ -36,7 +36,6 @@ mediapipe_ts_declaration( "audio_classifier_result.d.ts", ], deps = [ - "//mediapipe/tasks/web/audio/core:audio_task_options", "//mediapipe/tasks/web/components/containers:category", "//mediapipe/tasks/web/components/containers:classification_result", "//mediapipe/tasks/web/core", diff --git a/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts b/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts index e606019f..4e12780d 100644 --- a/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts +++ b/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts @@ -22,8 +22,8 @@ import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/b import {AudioTaskRunner} from '../../../../tasks/web/audio/core/audio_task_runner'; import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options'; import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result'; -import {TaskRunner} from '../../../../tasks/web/core/task_runner'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; +import {WasmModule} from '../../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource url import {AudioClassifierOptions} from './audio_classifier_options'; @@ -56,13 +56,12 @@ export class AudioClassifier extends AudioTaskRunner { * that either a path to the model asset or a model buffer needs to be * provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, audioClassifierOptions: AudioClassifierOptions): Promise { - const classifier = await TaskRunner.createInstance( - AudioClassifier, /* initializeCanvas= */ false, wasmFileset); - await classifier.setOptions(audioClassifierOptions); - return classifier; + return AudioTaskRunner.createInstance( + AudioClassifier, /* initializeCanvas= */ false, wasmFileset, + audioClassifierOptions); } /** @@ -75,8 +74,9 @@ export class AudioClassifier extends AudioTaskRunner { static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return AudioClassifier.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return AudioTaskRunner.createInstance( + AudioClassifier, /* initializeCanvas= */ false, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -86,20 +86,26 @@ export class AudioClassifier extends AudioTaskRunner { * Wasm binary and its loader. * @param modelAssetPath The path to the model asset. */ - static async createFromModelPath( + static createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return AudioClassifier.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return AudioTaskRunner.createInstance( + AudioClassifier, /* initializeCanvas= */ false, wasmFileset, + {baseOptions: {modelAssetPath}}); } - protected override get baseOptions(): BaseOptionsProto|undefined { - return this.options.getBaseOptions(); + constructor( + wasmModule: WasmModule, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) { + super(wasmModule, glCanvas); + this.options.setBaseOptions(new BaseOptionsProto()); } - protected override set baseOptions(proto: BaseOptionsProto|undefined) { + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; + } + + protected override set baseOptions(proto: BaseOptionsProto) { this.options.setBaseOptions(proto); } diff --git a/mediapipe/tasks/web/audio/audio_classifier/audio_classifier_options.d.ts b/mediapipe/tasks/web/audio/audio_classifier/audio_classifier_options.d.ts index 975b1e31..dc3c494b 100644 --- a/mediapipe/tasks/web/audio/audio_classifier/audio_classifier_options.d.ts +++ b/mediapipe/tasks/web/audio/audio_classifier/audio_classifier_options.d.ts @@ -14,9 +14,9 @@ * limitations under the License. */ -import {AudioTaskOptions} from '../../../../tasks/web/audio/core/audio_task_options'; import {ClassifierOptions} from '../../../../tasks/web/core/classifier_options'; +import {TaskRunnerOptions} from '../../../../tasks/web/core/task_runner_options'; /** Options to configure the MediaPipe Audio Classifier Task */ export declare interface AudioClassifierOptions extends ClassifierOptions, - AudioTaskOptions {} + TaskRunnerOptions {} diff --git a/mediapipe/tasks/web/audio/audio_embedder/BUILD b/mediapipe/tasks/web/audio/audio_embedder/BUILD index 1a66464b..0555bb63 100644 --- a/mediapipe/tasks/web/audio/audio_embedder/BUILD +++ b/mediapipe/tasks/web/audio/audio_embedder/BUILD @@ -36,7 +36,6 @@ mediapipe_ts_declaration( "audio_embedder_result.d.ts", ], deps = [ - "//mediapipe/tasks/web/audio/core:audio_task_options", "//mediapipe/tasks/web/components/containers:embedding_result", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:embedder_options", diff --git a/mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts b/mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts index c87aceab..d08eb479 100644 --- a/mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts +++ b/mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts @@ -25,7 +25,7 @@ import {convertEmbedderOptionsToProto} from '../../../../tasks/web/components/pr import {convertFromEmbeddingResultProto} from '../../../../tasks/web/components/processors/embedder_result'; import {computeCosineSimilarity} from '../../../../tasks/web/components/utils/cosine_similarity'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; -import {createMediaPipeLib, FileLocator} from '../../../../web/graph_runner/graph_runner'; +import {WasmModule} from '../../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource url import {AudioEmbedderOptions} from './audio_embedder_options'; @@ -58,23 +58,12 @@ export class AudioEmbedder extends AudioTaskRunner { * either a path to the TFLite model or the model itself needs to be * provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, audioEmbedderOptions: AudioEmbedderOptions): Promise { - // Create a file locator based on the loader options - const fileLocator: FileLocator = { - locateFile() { - // The only file we load is the Wasm binary - return wasmFileset.wasmBinaryPath.toString(); - } - }; - - const embedder = await createMediaPipeLib( - AudioEmbedder, wasmFileset.wasmLoaderPath, - /* assetLoaderScript= */ undefined, - /* glCanvas= */ undefined, fileLocator); - await embedder.setOptions(audioEmbedderOptions); - return embedder; + return AudioTaskRunner.createInstance( + AudioEmbedder, /* initializeCanvas= */ false, wasmFileset, + audioEmbedderOptions); } /** @@ -87,8 +76,9 @@ export class AudioEmbedder extends AudioTaskRunner { static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return AudioEmbedder.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return AudioTaskRunner.createInstance( + AudioEmbedder, /* initializeCanvas= */ false, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -98,20 +88,26 @@ export class AudioEmbedder extends AudioTaskRunner { * Wasm binary and its loader. * @param modelAssetPath The path to the TFLite model. */ - static async createFromModelPath( + static createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return AudioEmbedder.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return AudioTaskRunner.createInstance( + AudioEmbedder, /* initializeCanvas= */ false, wasmFileset, + {baseOptions: {modelAssetPath}}); } - protected override get baseOptions(): BaseOptionsProto|undefined { - return this.options.getBaseOptions(); + constructor( + wasmModule: WasmModule, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) { + super(wasmModule, glCanvas); + this.options.setBaseOptions(new BaseOptionsProto()); } - protected override set baseOptions(proto: BaseOptionsProto|undefined) { + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; + } + + protected override set baseOptions(proto: BaseOptionsProto) { this.options.setBaseOptions(proto); } diff --git a/mediapipe/tasks/web/audio/audio_embedder/audio_embedder_options.d.ts b/mediapipe/tasks/web/audio/audio_embedder/audio_embedder_options.d.ts index 98f412d0..ac22728a 100644 --- a/mediapipe/tasks/web/audio/audio_embedder/audio_embedder_options.d.ts +++ b/mediapipe/tasks/web/audio/audio_embedder/audio_embedder_options.d.ts @@ -14,9 +14,9 @@ * limitations under the License. */ -import {AudioTaskOptions} from '../../../../tasks/web/audio/core/audio_task_options'; import {EmbedderOptions} from '../../../../tasks/web/core/embedder_options'; +import {TaskRunnerOptions} from '../../../../tasks/web/core/task_runner_options'; /** Options to configure the MediaPipe Audio Embedder Task */ export declare interface AudioEmbedderOptions extends EmbedderOptions, - AudioTaskOptions {} + TaskRunnerOptions {} diff --git a/mediapipe/tasks/web/audio/core/BUILD b/mediapipe/tasks/web/audio/core/BUILD index 91ebbf52..9ab6c7be 100644 --- a/mediapipe/tasks/web/audio/core/BUILD +++ b/mediapipe/tasks/web/audio/core/BUILD @@ -1,24 +1,13 @@ # This package contains options shared by all MediaPipe Audio Tasks for Web. -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_declaration", "mediapipe_ts_library") +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library") package(default_visibility = ["//mediapipe/tasks:internal"]) -mediapipe_ts_declaration( - name = "audio_task_options", - srcs = ["audio_task_options.d.ts"], - deps = [ - "//mediapipe/tasks/web/core", - ], -) - mediapipe_ts_library( name = "audio_task_runner", srcs = ["audio_task_runner.ts"], deps = [ - ":audio_task_options", - "//mediapipe/tasks/cc/core/proto:base_options_jspb_proto", - "//mediapipe/tasks/web/components/processors:base_options", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:task_runner", ], diff --git a/mediapipe/tasks/web/audio/core/audio_task_options.d.ts b/mediapipe/tasks/web/audio/core/audio_task_options.d.ts deleted file mode 100644 index e3068625..00000000 --- a/mediapipe/tasks/web/audio/core/audio_task_options.d.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 The MediaPipe Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import {BaseOptions} from '../../../../tasks/web/core/base_options'; - -/** The options for configuring a MediaPipe Audio Task. */ -export declare interface AudioTaskOptions { - /** Options to configure the loading of the model assets. */ - baseOptions?: BaseOptions; -} diff --git a/mediapipe/tasks/web/audio/core/audio_task_runner.ts b/mediapipe/tasks/web/audio/core/audio_task_runner.ts index ceff3895..00cfe025 100644 --- a/mediapipe/tasks/web/audio/core/audio_task_runner.ts +++ b/mediapipe/tasks/web/audio/core/audio_task_runner.ts @@ -14,26 +14,13 @@ * limitations under the License. */ -import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/base_options_pb'; -import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options'; import {TaskRunner} from '../../../../tasks/web/core/task_runner'; - -import {AudioTaskOptions} from './audio_task_options'; +import {TaskRunnerOptions} from '../../../../tasks/web/core/task_runner_options'; /** Base class for all MediaPipe Audio Tasks. */ -export abstract class AudioTaskRunner extends TaskRunner { - protected abstract baseOptions?: BaseOptionsProto|undefined; +export abstract class AudioTaskRunner extends TaskRunner { private defaultSampleRate = 48000; - /** Configures the shared options of an audio task. */ - async setOptions(options: AudioTaskOptions): Promise { - this.baseOptions = this.baseOptions ?? new BaseOptionsProto(); - if (options.baseOptions) { - this.baseOptions = await convertBaseOptionsToProto( - options.baseOptions, this.baseOptions); - } - } - /** * Sets the sample rate for API calls that omit an explicit sample rate. * `48000` is used as a default if this method is not called. diff --git a/mediapipe/tasks/web/components/processors/BUILD b/mediapipe/tasks/web/components/processors/BUILD index 1b56bf4c..86e74392 100644 --- a/mediapipe/tasks/web/components/processors/BUILD +++ b/mediapipe/tasks/web/components/processors/BUILD @@ -17,7 +17,6 @@ mediapipe_ts_library( name = "classifier_result", srcs = ["classifier_result.ts"], deps = [ - "//mediapipe/framework/formats:classification_jspb_proto", "//mediapipe/tasks/cc/components/containers/proto:classifications_jspb_proto", "//mediapipe/tasks/web/components/containers:classification_result", ], diff --git a/mediapipe/tasks/web/components/processors/base_options.ts b/mediapipe/tasks/web/components/processors/base_options.ts index ac24a8db..16d56226 100644 --- a/mediapipe/tasks/web/components/processors/base_options.ts +++ b/mediapipe/tasks/web/components/processors/base_options.ts @@ -18,7 +18,7 @@ import {InferenceCalculatorOptions} from '../../../../calculators/tensor/inferen import {Acceleration} from '../../../../tasks/cc/core/proto/acceleration_pb'; import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/base_options_pb'; import {ExternalFile} from '../../../../tasks/cc/core/proto/external_file_pb'; -import {BaseOptions} from '../../../../tasks/web/core/base_options'; +import {BaseOptions} from '../../../../tasks/web/core/task_runner_options'; // The OSS JS API does not support the builder pattern. // tslint:disable:jspb-use-builder-pattern diff --git a/mediapipe/tasks/web/core/BUILD b/mediapipe/tasks/web/core/BUILD index d709e340..de429690 100644 --- a/mediapipe/tasks/web/core/BUILD +++ b/mediapipe/tasks/web/core/BUILD @@ -7,18 +7,18 @@ package(default_visibility = ["//mediapipe/tasks:internal"]) mediapipe_ts_declaration( name = "core", srcs = [ - "base_options.d.ts", + "task_runner_options.d.ts", "wasm_fileset.d.ts", ], ) mediapipe_ts_library( name = "task_runner", - srcs = [ - "task_runner.ts", - ], + srcs = ["task_runner.ts"], deps = [ ":core", + "//mediapipe/tasks/cc/core/proto:base_options_jspb_proto", + "//mediapipe/tasks/web/components/processors:base_options", "//mediapipe/web/graph_runner:graph_runner_image_lib_ts", "//mediapipe/web/graph_runner:graph_runner_ts", "//mediapipe/web/graph_runner:register_model_resources_graph_service_ts", diff --git a/mediapipe/tasks/web/core/classifier_options.d.ts b/mediapipe/tasks/web/core/classifier_options.d.ts index 1d804d62..08e7a766 100644 --- a/mediapipe/tasks/web/core/classifier_options.d.ts +++ b/mediapipe/tasks/web/core/classifier_options.d.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import {BaseOptions} from '../../../tasks/web/core/base_options'; - /** Options to configure a MediaPipe Classifier Task. */ export declare interface ClassifierOptions { /** diff --git a/mediapipe/tasks/web/core/embedder_options.d.ts b/mediapipe/tasks/web/core/embedder_options.d.ts index 3ec2a170..8669acfc 100644 --- a/mediapipe/tasks/web/core/embedder_options.d.ts +++ b/mediapipe/tasks/web/core/embedder_options.d.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import {BaseOptions} from '../../../tasks/web/core/base_options'; - /** Options to configure a MediaPipe Embedder Task */ export declare interface EmbedderOptions { /** diff --git a/mediapipe/tasks/web/core/task_runner.ts b/mediapipe/tasks/web/core/task_runner.ts index 4085be69..c2691fc7 100644 --- a/mediapipe/tasks/web/core/task_runner.ts +++ b/mediapipe/tasks/web/core/task_runner.ts @@ -14,6 +14,9 @@ * limitations under the License. */ +import {BaseOptions as BaseOptionsProto} from '../../../tasks/cc/core/proto/base_options_pb'; +import {convertBaseOptionsToProto} from '../../../tasks/web/components/processors/base_options'; +import {TaskRunnerOptions} from '../../../tasks/web/core/task_runner_options'; import {createMediaPipeLib, FileLocator, GraphRunner, WasmMediaPipeConstructor, WasmModule} from '../../../web/graph_runner/graph_runner'; import {SupportImage} from '../../../web/graph_runner/graph_runner_image_lib'; import {SupportModelResourcesGraphService} from '../../../web/graph_runner/register_model_resources_graph_service'; @@ -28,7 +31,9 @@ const WasmMediaPipeImageLib = SupportModelResourcesGraphService(SupportImage(GraphRunner)); /** Base class for all MediaPipe Tasks. */ -export abstract class TaskRunner extends WasmMediaPipeImageLib { +export abstract class TaskRunner extends + WasmMediaPipeImageLib { + protected abstract baseOptions: BaseOptionsProto; private processingErrors: Error[] = []; /** @@ -36,9 +41,10 @@ export abstract class TaskRunner extends WasmMediaPipeImageLib { * supported and loads the relevant WASM binary. * @return A fully instantiated instance of `T`. */ - protected static async createInstance( + protected static async createInstance, + O extends TaskRunnerOptions>( type: WasmMediaPipeConstructor, initializeCanvas: boolean, - fileset: WasmFileset): Promise { + fileset: WasmFileset, options: O): Promise { const fileLocator: FileLocator = { locateFile() { // The only file loaded with this mechanism is the Wasm binary @@ -46,19 +52,16 @@ export abstract class TaskRunner extends WasmMediaPipeImageLib { } }; - if (initializeCanvas) { - // Fall back to an OffscreenCanvas created by the GraphRunner if - // OffscreenCanvas is available - const canvas = typeof OffscreenCanvas === 'undefined' ? - document.createElement('canvas') : - undefined; - return createMediaPipeLib( - type, fileset.wasmLoaderPath, NO_ASSETS, canvas, fileLocator); - } else { - return createMediaPipeLib( - type, fileset.wasmLoaderPath, NO_ASSETS, /* glCanvas= */ null, - fileLocator); - } + // Initialize a canvas if requested. If OffscreenCanvas is availble, we + // let the graph runner initialize it by passing `undefined`. + const canvas = initializeCanvas ? (typeof OffscreenCanvas === 'undefined' ? + document.createElement('canvas') : + undefined) : + null; + const instance = await createMediaPipeLib( + type, fileset.wasmLoaderPath, NO_ASSETS, canvas, fileLocator); + await instance.setOptions(options); + return instance; } constructor( @@ -74,6 +77,14 @@ export abstract class TaskRunner extends WasmMediaPipeImageLib { this.registerModelResourcesGraphService(); } + /** Configures the shared options of a MediaPipe Task. */ + async setOptions(options: O): Promise { + if (options.baseOptions) { + this.baseOptions = await convertBaseOptionsToProto( + options.baseOptions, this.baseOptions); + } + } + /** * Takes the raw data from a MediaPipe graph, and passes it to C++ to be run * over the video stream. Will replace the previously running MediaPipe graph, diff --git a/mediapipe/tasks/web/core/base_options.d.ts b/mediapipe/tasks/web/core/task_runner_options.d.ts similarity index 85% rename from mediapipe/tasks/web/core/base_options.d.ts rename to mediapipe/tasks/web/core/task_runner_options.d.ts index 86635b8c..aa0b4a02 100644 --- a/mediapipe/tasks/web/core/base_options.d.ts +++ b/mediapipe/tasks/web/core/task_runner_options.d.ts @@ -16,7 +16,7 @@ // Placeholder for internal dependency on trusted resource url -/** Options to configure MediaPipe Tasks in general. */ +/** Options to configure MediaPipe model loading and processing. */ export declare interface BaseOptions { /** * The model path to the model asset file. Only one of `modelAssetPath` or @@ -33,3 +33,9 @@ export declare interface BaseOptions { /** Overrides the default backend to use for the provided model. */ delegate?: 'cpu'|'gpu'|undefined; } + +/** Options to configure MediaPipe Tasks in general. */ +export declare interface TaskRunnerOptions { + /** Options to configure the loading of the model assets. */ + baseOptions?: BaseOptions; +} diff --git a/mediapipe/tasks/web/text/core/BUILD b/mediapipe/tasks/web/text/core/BUILD deleted file mode 100644 index 3e7faec9..00000000 --- a/mediapipe/tasks/web/text/core/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -# This package contains options shared by all MediaPipe Texxt Tasks for Web. - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_declaration") - -package(default_visibility = ["//mediapipe/tasks:internal"]) - -mediapipe_ts_declaration( - name = "text_task_options", - srcs = ["text_task_options.d.ts"], - deps = ["//mediapipe/tasks/web/core"], -) diff --git a/mediapipe/tasks/web/text/core/text_task_options.d.ts b/mediapipe/tasks/web/text/core/text_task_options.d.ts deleted file mode 100644 index 4874e35b..00000000 --- a/mediapipe/tasks/web/text/core/text_task_options.d.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 The MediaPipe Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import {BaseOptions} from '../../../../tasks/web/core/base_options'; - -/** The options for configuring a MediaPipe Text task. */ -export declare interface TextTaskOptions { - /** Options to configure the loading of the model assets. */ - baseOptions?: BaseOptions; -} diff --git a/mediapipe/tasks/web/text/text_classifier/BUILD b/mediapipe/tasks/web/text/text_classifier/BUILD index f3d272da..2a7de21d 100644 --- a/mediapipe/tasks/web/text/text_classifier/BUILD +++ b/mediapipe/tasks/web/text/text_classifier/BUILD @@ -17,15 +17,16 @@ mediapipe_ts_library( "//mediapipe/framework:calculator_jspb_proto", "//mediapipe/framework:calculator_options_jspb_proto", "//mediapipe/tasks/cc/components/containers/proto:classifications_jspb_proto", + "//mediapipe/tasks/cc/core/proto:base_options_jspb_proto", "//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_jspb_proto", "//mediapipe/tasks/web/components/containers:category", "//mediapipe/tasks/web/components/containers:classification_result", - "//mediapipe/tasks/web/components/processors:base_options", "//mediapipe/tasks/web/components/processors:classifier_options", "//mediapipe/tasks/web/components/processors:classifier_result", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:classifier_options", "//mediapipe/tasks/web/core:task_runner", + "//mediapipe/web/graph_runner:graph_runner_ts", ], ) @@ -38,7 +39,7 @@ mediapipe_ts_declaration( deps = [ "//mediapipe/tasks/web/components/containers:category", "//mediapipe/tasks/web/components/containers:classification_result", + "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:classifier_options", - "//mediapipe/tasks/web/text/core:text_task_options", ], ) diff --git a/mediapipe/tasks/web/text/text_classifier/text_classifier.ts b/mediapipe/tasks/web/text/text_classifier/text_classifier.ts index 197869a3..bd2a207c 100644 --- a/mediapipe/tasks/web/text/text_classifier/text_classifier.ts +++ b/mediapipe/tasks/web/text/text_classifier/text_classifier.ts @@ -17,12 +17,13 @@ import {CalculatorGraphConfig} from '../../../../framework/calculator_pb'; import {CalculatorOptions} from '../../../../framework/calculator_options_pb'; import {ClassificationResult} from '../../../../tasks/cc/components/containers/proto/classifications_pb'; +import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/base_options_pb'; import {TextClassifierGraphOptions} from '../../../../tasks/cc/text/text_classifier/proto/text_classifier_graph_options_pb'; -import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options'; import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options'; import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result'; import {TaskRunner} from '../../../../tasks/web/core/task_runner'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; +import {WasmModule} from '../../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource url import {TextClassifierOptions} from './text_classifier_options'; @@ -40,7 +41,7 @@ const TEXT_CLASSIFIER_GRAPH = // tslint:disable:jspb-use-builder-pattern /** Performs Natural Language classification. */ -export class TextClassifier extends TaskRunner { +export class TextClassifier extends TaskRunner { private classificationResult: TextClassifierResult = {classifications: []}; private readonly options = new TextClassifierGraphOptions(); @@ -53,13 +54,12 @@ export class TextClassifier extends TaskRunner { * either a path to the TFLite model or the model itself needs to be * provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, textClassifierOptions: TextClassifierOptions): Promise { - const classifier = await TaskRunner.createInstance( - TextClassifier, /* initializeCanvas= */ false, wasmFileset); - await classifier.setOptions(textClassifierOptions); - return classifier; + return TaskRunner.createInstance( + TextClassifier, /* initializeCanvas= */ false, wasmFileset, + textClassifierOptions); } /** @@ -72,8 +72,9 @@ export class TextClassifier extends TaskRunner { static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return TextClassifier.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return TaskRunner.createInstance( + TextClassifier, /* initializeCanvas= */ false, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -83,13 +84,19 @@ export class TextClassifier extends TaskRunner { * Wasm binary and its loader. * @param modelAssetPath The path to the model asset. */ - static async createFromModelPath( + static createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return TextClassifier.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return TaskRunner.createInstance( + TextClassifier, /* initializeCanvas= */ false, wasmFileset, + {baseOptions: {modelAssetPath}}); + } + + constructor( + wasmModule: WasmModule, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) { + super(wasmModule, glCanvas); + this.options.setBaseOptions(new BaseOptionsProto()); } /** @@ -101,18 +108,20 @@ export class TextClassifier extends TaskRunner { * * @param options The options for the text classifier. */ - async setOptions(options: TextClassifierOptions): Promise { - if (options.baseOptions) { - const baseOptionsProto = await convertBaseOptionsToProto( - options.baseOptions, this.options.getBaseOptions()); - this.options.setBaseOptions(baseOptionsProto); - } - + override async setOptions(options: TextClassifierOptions): Promise { + await super.setOptions(options); this.options.setClassifierOptions(convertClassifierOptionsToProto( options, this.options.getClassifierOptions())); this.refreshGraph(); } + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; + } + + protected override set baseOptions(proto: BaseOptionsProto) { + this.options.setBaseOptions(proto); + } /** * Performs Natural Language classification on the provided text and waits diff --git a/mediapipe/tasks/web/text/text_classifier/text_classifier_options.d.ts b/mediapipe/tasks/web/text/text_classifier/text_classifier_options.d.ts index b50767e1..25592deb 100644 --- a/mediapipe/tasks/web/text/text_classifier/text_classifier_options.d.ts +++ b/mediapipe/tasks/web/text/text_classifier/text_classifier_options.d.ts @@ -15,8 +15,8 @@ */ import {ClassifierOptions} from '../../../../tasks/web/core/classifier_options'; -import {TextTaskOptions} from '../../../../tasks/web/text/core/text_task_options'; +import {TaskRunnerOptions} from '../../../../tasks/web/core/task_runner_options'; /** Options to configure the MediaPipe Text Classifier Task */ export declare interface TextClassifierOptions extends ClassifierOptions, - TextTaskOptions {} + TaskRunnerOptions {} diff --git a/mediapipe/tasks/web/text/text_embedder/BUILD b/mediapipe/tasks/web/text/text_embedder/BUILD index b858f6b8..17d10525 100644 --- a/mediapipe/tasks/web/text/text_embedder/BUILD +++ b/mediapipe/tasks/web/text/text_embedder/BUILD @@ -17,15 +17,16 @@ mediapipe_ts_library( "//mediapipe/framework:calculator_jspb_proto", "//mediapipe/framework:calculator_options_jspb_proto", "//mediapipe/tasks/cc/components/containers/proto:embeddings_jspb_proto", + "//mediapipe/tasks/cc/core/proto:base_options_jspb_proto", "//mediapipe/tasks/cc/text/text_embedder/proto:text_embedder_graph_options_jspb_proto", "//mediapipe/tasks/web/components/containers:embedding_result", - "//mediapipe/tasks/web/components/processors:base_options", "//mediapipe/tasks/web/components/processors:embedder_options", "//mediapipe/tasks/web/components/processors:embedder_result", "//mediapipe/tasks/web/components/utils:cosine_similarity", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:embedder_options", "//mediapipe/tasks/web/core:task_runner", + "//mediapipe/web/graph_runner:graph_runner_ts", ], ) @@ -39,6 +40,5 @@ mediapipe_ts_declaration( "//mediapipe/tasks/web/components/containers:embedding_result", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:embedder_options", - "//mediapipe/tasks/web/text/core:text_task_options", ], ) diff --git a/mediapipe/tasks/web/text/text_embedder/text_embedder.ts b/mediapipe/tasks/web/text/text_embedder/text_embedder.ts index 511fd241..d2899fbe 100644 --- a/mediapipe/tasks/web/text/text_embedder/text_embedder.ts +++ b/mediapipe/tasks/web/text/text_embedder/text_embedder.ts @@ -17,14 +17,15 @@ import {CalculatorGraphConfig} from '../../../../framework/calculator_pb'; import {CalculatorOptions} from '../../../../framework/calculator_options_pb'; import {EmbeddingResult} from '../../../../tasks/cc/components/containers/proto/embeddings_pb'; +import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/base_options_pb'; import {TextEmbedderGraphOptions as TextEmbedderGraphOptionsProto} from '../../../../tasks/cc/text/text_embedder/proto/text_embedder_graph_options_pb'; import {Embedding} from '../../../../tasks/web/components/containers/embedding_result'; -import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options'; import {convertEmbedderOptionsToProto} from '../../../../tasks/web/components/processors/embedder_options'; import {convertFromEmbeddingResultProto} from '../../../../tasks/web/components/processors/embedder_result'; import {computeCosineSimilarity} from '../../../../tasks/web/components/utils/cosine_similarity'; import {TaskRunner} from '../../../../tasks/web/core/task_runner'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; +import {WasmModule} from '../../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource url import {TextEmbedderOptions} from './text_embedder_options'; @@ -44,7 +45,7 @@ const TEXT_EMBEDDER_CALCULATOR = /** * Performs embedding extraction on text. */ -export class TextEmbedder extends TaskRunner { +export class TextEmbedder extends TaskRunner { private embeddingResult: TextEmbedderResult = {embeddings: []}; private readonly options = new TextEmbedderGraphOptionsProto(); @@ -57,13 +58,12 @@ export class TextEmbedder extends TaskRunner { * either a path to the TFLite model or the model itself needs to be * provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, textEmbedderOptions: TextEmbedderOptions): Promise { - const embedder = await TaskRunner.createInstance( - TextEmbedder, /* initializeCanvas= */ false, wasmFileset); - await embedder.setOptions(textEmbedderOptions); - return embedder; + return TaskRunner.createInstance( + TextEmbedder, /* initializeCanvas= */ false, wasmFileset, + textEmbedderOptions); } /** @@ -76,8 +76,9 @@ export class TextEmbedder extends TaskRunner { static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return TextEmbedder.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return TaskRunner.createInstance( + TextEmbedder, /* initializeCanvas= */ false, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -87,13 +88,19 @@ export class TextEmbedder extends TaskRunner { * Wasm binary and its loader. * @param modelAssetPath The path to the TFLite model. */ - static async createFromModelPath( + static createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return TextEmbedder.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return TaskRunner.createInstance( + TextEmbedder, /* initializeCanvas= */ false, wasmFileset, + {baseOptions: {modelAssetPath}}); + } + + constructor( + wasmModule: WasmModule, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) { + super(wasmModule, glCanvas); + this.options.setBaseOptions(new BaseOptionsProto()); } /** @@ -105,17 +112,21 @@ export class TextEmbedder extends TaskRunner { * * @param options The options for the text embedder. */ - async setOptions(options: TextEmbedderOptions): Promise { - if (options.baseOptions) { - const baseOptionsProto = await convertBaseOptionsToProto( - options.baseOptions, this.options.getBaseOptions()); - this.options.setBaseOptions(baseOptionsProto); - } + override async setOptions(options: TextEmbedderOptions): Promise { + await super.setOptions(options); this.options.setEmbedderOptions(convertEmbedderOptionsToProto( options, this.options.getEmbedderOptions())); this.refreshGraph(); } + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; + } + + protected override set baseOptions(proto: BaseOptionsProto) { + this.options.setBaseOptions(proto); + } + /** * Performs embeding extraction on the provided text and waits synchronously * for the response. diff --git a/mediapipe/tasks/web/text/text_embedder/text_embedder_options.d.ts b/mediapipe/tasks/web/text/text_embedder/text_embedder_options.d.ts index 9ea57030..7689ee0c 100644 --- a/mediapipe/tasks/web/text/text_embedder/text_embedder_options.d.ts +++ b/mediapipe/tasks/web/text/text_embedder/text_embedder_options.d.ts @@ -15,8 +15,8 @@ */ import {EmbedderOptions} from '../../../../tasks/web/core/embedder_options'; -import {TextTaskOptions} from '../../../../tasks/web/text/core/text_task_options'; +import {TaskRunnerOptions} from '../../../../tasks/web/core/task_runner_options'; /** Options to configure the MediaPipe Text Embedder Task */ export declare interface TextEmbedderOptions extends EmbedderOptions, - TextTaskOptions {} + TaskRunnerOptions {} diff --git a/mediapipe/tasks/web/vision/core/BUILD b/mediapipe/tasks/web/vision/core/BUILD index 1d8944f1..b389a9b0 100644 --- a/mediapipe/tasks/web/vision/core/BUILD +++ b/mediapipe/tasks/web/vision/core/BUILD @@ -17,8 +17,6 @@ mediapipe_ts_library( srcs = ["vision_task_runner.ts"], deps = [ ":vision_task_options", - "//mediapipe/tasks/cc/core/proto:base_options_jspb_proto", - "//mediapipe/tasks/web/components/processors:base_options", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:task_runner", "//mediapipe/web/graph_runner:graph_runner_ts", diff --git a/mediapipe/tasks/web/vision/core/vision_task_options.d.ts b/mediapipe/tasks/web/vision/core/vision_task_options.d.ts index e04eb659..76c0177a 100644 --- a/mediapipe/tasks/web/vision/core/vision_task_options.d.ts +++ b/mediapipe/tasks/web/vision/core/vision_task_options.d.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import {BaseOptions} from '../../../../tasks/web/core/base_options'; +import {TaskRunnerOptions} from '../../../../tasks/web/core/task_runner_options'; /** * The two running modes of a vision task. @@ -23,12 +23,8 @@ import {BaseOptions} from '../../../../tasks/web/core/base_options'; */ export type RunningMode = 'image'|'video'; - /** The options for configuring a MediaPipe vision task. */ -export declare interface VisionTaskOptions { - /** Options to configure the loading of the model assets. */ - baseOptions?: BaseOptions; - +export declare interface VisionTaskOptions extends TaskRunnerOptions { /** * The running mode of the task. Default to the image mode. * Vision tasks have two running modes: diff --git a/mediapipe/tasks/web/vision/core/vision_task_runner.ts b/mediapipe/tasks/web/vision/core/vision_task_runner.ts index 79ff4515..78b4859f 100644 --- a/mediapipe/tasks/web/vision/core/vision_task_runner.ts +++ b/mediapipe/tasks/web/vision/core/vision_task_runner.ts @@ -14,24 +14,17 @@ * limitations under the License. */ -import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/base_options_pb'; -import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options'; import {TaskRunner} from '../../../../tasks/web/core/task_runner'; import {ImageSource} from '../../../../web/graph_runner/graph_runner'; import {VisionTaskOptions} from './vision_task_options'; /** Base class for all MediaPipe Vision Tasks. */ -export abstract class VisionTaskRunner extends TaskRunner { - protected abstract baseOptions?: BaseOptionsProto|undefined; - +export abstract class VisionTaskRunner extends + TaskRunner { /** Configures the shared options of a vision task. */ - async setOptions(options: VisionTaskOptions): Promise { - this.baseOptions = this.baseOptions ?? new BaseOptionsProto(); - if (options.baseOptions) { - this.baseOptions = await convertBaseOptionsToProto( - options.baseOptions, this.baseOptions); - } + override async setOptions(options: VisionTaskOptions): Promise { + await super.setOptions(options); if ('runningMode' in options) { const useStreamMode = !!options.runningMode && options.runningMode !== 'image'; diff --git a/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts b/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts index 15b6acb1..8baee5ce 100644 --- a/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts +++ b/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts @@ -88,14 +88,13 @@ export class GestureRecognizer extends * Note that either a path to the model asset or a model buffer needs to * be provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, gestureRecognizerOptions: GestureRecognizerOptions): Promise { - const recognizer = await VisionTaskRunner.createInstance( - GestureRecognizer, /* initializeCanvas= */ true, wasmFileset); - await recognizer.setOptions(gestureRecognizerOptions); - return recognizer; + return VisionTaskRunner.createInstance( + GestureRecognizer, /* initializeCanvas= */ true, wasmFileset, + gestureRecognizerOptions); } /** @@ -108,8 +107,9 @@ export class GestureRecognizer extends static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return GestureRecognizer.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return VisionTaskRunner.createInstance( + GestureRecognizer, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -119,13 +119,12 @@ export class GestureRecognizer extends * Wasm binary and its loader. * @param modelAssetPath The path to the model asset. */ - static async createFromModelPath( + static createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return GestureRecognizer.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return VisionTaskRunner.createInstance( + GestureRecognizer, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetPath}}); } constructor( @@ -134,6 +133,7 @@ export class GestureRecognizer extends super(wasmModule, glCanvas); this.options = new GestureRecognizerGraphOptions(); + this.options.setBaseOptions(new BaseOptionsProto()); this.handLandmarkerGraphOptions = new HandLandmarkerGraphOptions(); this.options.setHandLandmarkerGraphOptions(this.handLandmarkerGraphOptions); this.handLandmarksDetectorGraphOptions = @@ -151,11 +151,11 @@ export class GestureRecognizer extends this.initDefaults(); } - protected override get baseOptions(): BaseOptionsProto|undefined { - return this.options.getBaseOptions(); + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; } - protected override set baseOptions(proto: BaseOptionsProto|undefined) { + protected override set baseOptions(proto: BaseOptionsProto) { this.options.setBaseOptions(proto); } diff --git a/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts b/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts index c657275b..263ed4b4 100644 --- a/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts +++ b/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts @@ -77,13 +77,12 @@ export class HandLandmarker extends VisionTaskRunner { * Note that either a path to the model asset or a model buffer needs to * be provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, handLandmarkerOptions: HandLandmarkerOptions): Promise { - const landmarker = await VisionTaskRunner.createInstance( - HandLandmarker, /* initializeCanvas= */ true, wasmFileset); - await landmarker.setOptions(handLandmarkerOptions); - return landmarker; + return VisionTaskRunner.createInstance( + HandLandmarker, /* initializeCanvas= */ true, wasmFileset, + handLandmarkerOptions); } /** @@ -96,8 +95,9 @@ export class HandLandmarker extends VisionTaskRunner { static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return HandLandmarker.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return VisionTaskRunner.createInstance( + HandLandmarker, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -107,13 +107,12 @@ export class HandLandmarker extends VisionTaskRunner { * Wasm binary and its loader. * @param modelAssetPath The path to the model asset. */ - static async createFromModelPath( + static createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return HandLandmarker.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return VisionTaskRunner.createInstance( + HandLandmarker, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetPath}}); } constructor( @@ -122,6 +121,7 @@ export class HandLandmarker extends VisionTaskRunner { super(wasmModule, glCanvas); this.options = new HandLandmarkerGraphOptions(); + this.options.setBaseOptions(new BaseOptionsProto()); this.handLandmarksDetectorGraphOptions = new HandLandmarksDetectorGraphOptions(); this.options.setHandLandmarksDetectorGraphOptions( @@ -132,11 +132,11 @@ export class HandLandmarker extends VisionTaskRunner { this.initDefaults(); } - protected override get baseOptions(): BaseOptionsProto|undefined { - return this.options.getBaseOptions(); + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; } - protected override set baseOptions(proto: BaseOptionsProto|undefined) { + protected override set baseOptions(proto: BaseOptionsProto) { this.options.setBaseOptions(proto); } diff --git a/mediapipe/tasks/web/vision/image_classifier/image_classifier.ts b/mediapipe/tasks/web/vision/image_classifier/image_classifier.ts index 185ddf9e..90dbf979 100644 --- a/mediapipe/tasks/web/vision/image_classifier/image_classifier.ts +++ b/mediapipe/tasks/web/vision/image_classifier/image_classifier.ts @@ -23,7 +23,7 @@ import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/ import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner'; -import {ImageSource} from '../../../../web/graph_runner/graph_runner'; +import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource url import {ImageClassifierOptions} from './image_classifier_options'; @@ -55,13 +55,12 @@ export class ImageClassifier extends VisionTaskRunner { * that either a path to the model asset or a model buffer needs to be * provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, imageClassifierOptions: ImageClassifierOptions): Promise { - const classifier = await VisionTaskRunner.createInstance( - ImageClassifier, /* initializeCanvas= */ true, wasmFileset); - await classifier.setOptions(imageClassifierOptions); - return classifier; + return VisionTaskRunner.createInstance( + ImageClassifier, /* initializeCanvas= */ true, wasmFileset, + imageClassifierOptions); } /** @@ -74,8 +73,9 @@ export class ImageClassifier extends VisionTaskRunner { static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return ImageClassifier.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return VisionTaskRunner.createInstance( + ImageClassifier, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -85,20 +85,26 @@ export class ImageClassifier extends VisionTaskRunner { * Wasm binary and its loader. * @param modelAssetPath The path to the model asset. */ - static async createFromModelPath( + static createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return ImageClassifier.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return VisionTaskRunner.createInstance( + ImageClassifier, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetPath}}); } - protected override get baseOptions(): BaseOptionsProto|undefined { - return this.options.getBaseOptions(); + constructor( + wasmModule: WasmModule, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) { + super(wasmModule, glCanvas); + this.options.setBaseOptions(new BaseOptionsProto()); } - protected override set baseOptions(proto: BaseOptionsProto|undefined) { + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; + } + + protected override set baseOptions(proto: BaseOptionsProto) { this.options.setBaseOptions(proto); } diff --git a/mediapipe/tasks/web/vision/image_embedder/image_embedder.ts b/mediapipe/tasks/web/vision/image_embedder/image_embedder.ts index 91352e93..55933265 100644 --- a/mediapipe/tasks/web/vision/image_embedder/image_embedder.ts +++ b/mediapipe/tasks/web/vision/image_embedder/image_embedder.ts @@ -25,7 +25,7 @@ import {convertFromEmbeddingResultProto} from '../../../../tasks/web/components/ import {computeCosineSimilarity} from '../../../../tasks/web/components/utils/cosine_similarity'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner'; -import {ImageSource} from '../../../../web/graph_runner/graph_runner'; +import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource url import {ImageEmbedderOptions} from './image_embedder_options'; @@ -57,13 +57,12 @@ export class ImageEmbedder extends VisionTaskRunner { * either a path to the TFLite model or the model itself needs to be * provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, imageEmbedderOptions: ImageEmbedderOptions): Promise { - const embedder = await VisionTaskRunner.createInstance( - ImageEmbedder, /* initializeCanvas= */ true, wasmFileset); - await embedder.setOptions(imageEmbedderOptions); - return embedder; + return VisionTaskRunner.createInstance( + ImageEmbedder, /* initializeCanvas= */ true, wasmFileset, + imageEmbedderOptions); } /** @@ -76,8 +75,9 @@ export class ImageEmbedder extends VisionTaskRunner { static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return ImageEmbedder.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return VisionTaskRunner.createInstance( + ImageEmbedder, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -87,20 +87,26 @@ export class ImageEmbedder extends VisionTaskRunner { * Wasm binary and its loader. * @param modelAssetPath The path to the TFLite model. */ - static async createFromModelPath( + static createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return ImageEmbedder.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return VisionTaskRunner.createInstance( + ImageEmbedder, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetPath}}); } - protected override get baseOptions(): BaseOptionsProto|undefined { - return this.options.getBaseOptions(); + constructor( + wasmModule: WasmModule, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) { + super(wasmModule, glCanvas); + this.options.setBaseOptions(new BaseOptionsProto()); } - protected override set baseOptions(proto: BaseOptionsProto|undefined) { + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; + } + + protected override set baseOptions(proto: BaseOptionsProto) { this.options.setBaseOptions(proto); } diff --git a/mediapipe/tasks/web/vision/object_detector/object_detector.ts b/mediapipe/tasks/web/vision/object_detector/object_detector.ts index 7711c39e..03171003 100644 --- a/mediapipe/tasks/web/vision/object_detector/object_detector.ts +++ b/mediapipe/tasks/web/vision/object_detector/object_detector.ts @@ -21,7 +21,7 @@ import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/b import {ObjectDetectorOptions as ObjectDetectorOptionsProto} from '../../../../tasks/cc/vision/object_detector/proto/object_detector_options_pb'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner'; -import {ImageSource} from '../../../../web/graph_runner/graph_runner'; +import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource url import {ObjectDetectorOptions} from './object_detector_options'; @@ -54,13 +54,12 @@ export class ObjectDetector extends VisionTaskRunner { * either a path to the model asset or a model buffer needs to be * provided (via `baseOptions`). */ - static async createFromOptions( + static createFromOptions( wasmFileset: WasmFileset, objectDetectorOptions: ObjectDetectorOptions): Promise { - const detector = await VisionTaskRunner.createInstance( - ObjectDetector, /* initializeCanvas= */ true, wasmFileset); - await detector.setOptions(objectDetectorOptions); - return detector; + return VisionTaskRunner.createInstance( + ObjectDetector, /* initializeCanvas= */ true, wasmFileset, + objectDetectorOptions); } /** @@ -73,8 +72,9 @@ export class ObjectDetector extends VisionTaskRunner { static createFromModelBuffer( wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise { - return ObjectDetector.createFromOptions( - wasmFileset, {baseOptions: {modelAssetBuffer}}); + return VisionTaskRunner.createInstance( + ObjectDetector, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetBuffer}}); } /** @@ -87,17 +87,23 @@ export class ObjectDetector extends VisionTaskRunner { static async createFromModelPath( wasmFileset: WasmFileset, modelAssetPath: string): Promise { - const response = await fetch(modelAssetPath.toString()); - const graphData = await response.arrayBuffer(); - return ObjectDetector.createFromModelBuffer( - wasmFileset, new Uint8Array(graphData)); + return VisionTaskRunner.createInstance( + ObjectDetector, /* initializeCanvas= */ true, wasmFileset, + {baseOptions: {modelAssetPath}}); } - protected override get baseOptions(): BaseOptionsProto|undefined { - return this.options.getBaseOptions(); + constructor( + wasmModule: WasmModule, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) { + super(wasmModule, glCanvas); + this.options.setBaseOptions(new BaseOptionsProto()); } - protected override set baseOptions(proto: BaseOptionsProto|undefined) { + protected override get baseOptions(): BaseOptionsProto { + return this.options.getBaseOptions()!; + } + + protected override set baseOptions(proto: BaseOptionsProto) { this.options.setBaseOptions(proto); }