Resize bicubic op (#59)

* started resize bicubic

* started padding algorithm for bicubic forward pass in cpu backend

* started padding algorithm for bicubic forward pass in cpu backend

* Mark all calls to 'op()' as pure (tensorflow#7155)

Mark calls to the `op()` function that creates the exported op as pure by using [`/* @__PURE__ */` annotations](https://esbuild.github.io/api/#ignore-annotations) (this also works for Rollup, but I can't find the docs). This comment instructs bundlers that the function call has no side-effects, so it can be removed if the result is not used.

This is okay for the `op` function because, although it references ENGINE, it does so [in a closure](https://github.com/tensorflow/tfjs/blob/master/tfjs-core/src/ops/operation.ts#L48-L61) that it never calls, so while its return value may cause side effects when called, it itself does not.

This has no immediate effect because we still maintain a list of `sideEffects` in the package.json, but it is a step towards removing that list.

Co-authored-by: Linchenn <40653845+Linchenn@users.noreply.github.com>

* need to fix padding algo

* Update rules_python to 0.16.1 (tensorflow#7160)

This update includes new lock files for pypi packages that make sure their versions don't change between builds. These lock files can be generated with the update_locked_deps.sh script.

As part of this update, the PR pins flax to 0.6.2.

Additionally, python dependencies will only be fetched when a build requires them, so first time javascript-only builds should see a speedup.

* Register optimizers in a centralized location (tensorflow#7157)

Optimizers are currently registered in the same file they are defined in, which is a side effect. This would make it impossible to tree-shake them in a custom bundle when `sideEffects` is removed from the package.json.

This PR moves Optimizer registration to the `register_optimizers.ts` file. The files that the Optimizers are defined in no longer have side effects. To exclude Optimizers from a custom bundle, the `registerOptimizers` function is called from `index.ts`. Custom bundles replace `index.ts` with a different file that does not call this function.

* Simplify how Optimizers are re-exported in train.ts (tensorflow#7156)

`train.ts` exports optimizers by copying them from the `OptimizerConstructors` class onto a `train` object. This is unnecessary because the `OptimizerConstructors` class constructor is a subtype of the `train` object's type (i.e. it has all the properties that `train` has). Instead of creating a new `train` object, this PR re-exports `OptimizerConstructors` as `train`.

This has no direct effect now, but if / when re remove the `sideEffects` field from `package.json`, it helps some bundlers (esbuild) do tree-shaking.

* Use static getters to get optimizer class names (tensorflow#7168)

Each `Optimizer` lists its class name as a static property of the class so it can be serialized and deserialized. This prevents the class from being tree-shaken because bundlers will compile it like this:

```
class SomeOptimizer {
  ...
}

// The bundler can not remove this assignment because
// SomeOptimizer.className could be a setter with a side effect.
SomeOptimizer.className = 'SomeOptimizer';
```

This PR uses a static getter for the class name instead, which bundlers can tree-shake properly.

* need corners

* padding is functional

* debugging padding tool for multiple channels

Co-authored-by: Matthew Soulanille <msoulanille@google.com>
Co-authored-by: Linchenn <40653845+Linchenn@users.noreply.github.com>

WORKSPACE

@@ -211,34 +211,46 @@ http_archive(
 
 http_archive(
     name = "rules_python",
-    sha256 = "5fa3c738d33acca3b97622a13a741129f67ef43f5fdfcec63b29374cc0574c29",
-    strip_prefix = "rules_python-0.9.0",
-    url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.9.0.tar.gz",
+    sha256 = "497ca47374f48c8b067d786b512ac10a276211810f4a580178ee9b9ad139323a",
+    strip_prefix = "rules_python-0.16.1",
+    url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.16.1.tar.gz",
 )
 
 load("@rules_python//python:repositories.bzl", "python_register_toolchains")
 
+# TODO(mattSoulanille): Change the docker so it doesn't run as root?
+# https://github.com/bazelbuild/rules_python/pull/713
+# https://github.com/GoogleCloudPlatform/cloud-builders/issues/641
 python_register_toolchains(
     name = "python3_8",
+    ignore_root_user_error = True,
     # Available versions are listed in @rules_python//python:versions.bzl.
     python_version = "3.8",
 )
 
 load("@python3_8//:defs.bzl", "interpreter")
-load("@rules_python//python:pip.bzl", "pip_install")
+load("@rules_python//python:pip.bzl", "pip_parse")
 
-pip_install(
-    name = "tensorflowjs_dev_deps",
+pip_parse(
+    name = "tensorflowjs_deps",
     python_interpreter_target = interpreter,
-    requirements = "@//tfjs-converter/python:requirements-dev.txt",
+    requirements_lock = "@//tfjs-converter/python:requirements_lock.txt",
 )
 
-pip_install(
-    name = "tensorflowjs_deps",
+load("@tensorflowjs_deps//:requirements.bzl", install_tfjs_deps = "install_deps")
+
+install_tfjs_deps()
+
+pip_parse(
+    name = "tensorflowjs_dev_deps",
     python_interpreter_target = interpreter,
-    requirements = "@//tfjs-converter/python:requirements.txt",
+    requirements_lock = "@//tfjs-converter/python:requirements-dev_lock.txt",
 )
 
+load("@tensorflowjs_dev_deps//:requirements.bzl", install_tfjs_dev_deps = "install_deps")
+
+install_tfjs_dev_deps()
+
 load("//tfjs-tflite:tflite_repositories.bzl", "tflite_repositories")
 
 tflite_repositories()

tfjs-backend-cpu/src/kernels/ResizeBicubic.ts

@@ -0,0 +1,131 @@
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+import {KernelConfig, KernelFunc, ResizeBicubic, ResizeBicubicAttrs, ResizeBicubicInputs, TensorInfo, TypedArray, util} from '@tensorflow/tfjs-core';
+
+import {MathBackendCPU} from '../backend_cpu';
+import {assertNotComplex} from '../cpu_util';
+
+export function resizeBicubic(args: {
+  inputs: ResizeBicubicInputs,
+  backend: MathBackendCPU,
+  attrs: ResizeBicubicAttrs
+
+}): TensorInfo {
+  const {inputs, backend, attrs} = args;
+  const {images} = inputs;
+  const {alignCorners, halfPixelCenters, size} = attrs;
+  console.log(`alignCorners ${alignCorners}`)
+  console.log(`halfPixelCenters ${halfPixelCenters}`)
+
+  assertNotComplex(images, 'resizeBicubic');
+  const imagesStrides = util.computeStrides(images.shape);
+  const [newHeight, newWidth] = size;
+  const [batch, oldHeight, oldWidth, numChannels] = images.shape;
+
+  const xValues = backend.data.get(images.dataId).values as TypedArray;
+  console.log(`xValues ${xValues}`)
+  console.log(`imagesStrides ${imagesStrides}`)
+
+  function indexToCoordinates(point: number, imagesStrides: number[]): number[] {
+    const rowPoint = Math.floor(point / imagesStrides[1])
+    const colPoint = point % imagesStrides[1]
+    return [rowPoint, colPoint]
+  }
+
+  function coordinatesToIndex(row: number, col:number, imagesStrides: number[]) {
+    return (row * imagesStrides[1]) + col
+  }
+
+  function padInputImage(img: TypedArray): Float32Array {
+    const pad = 2;
+    const paddedMat = new Float32Array(util.sizeFromShape([batch, oldHeight + pad*2, oldWidth + pad*2, numChannels]));
+    const paddedStrides = util.computeStrides([batch, oldHeight + pad*2, oldWidth + pad*2, numChannels]);
+    const [imgRow, imgCol] = [(imagesStrides[0] / imagesStrides[1]), imagesStrides[1]];
+    const maxLen = paddedMat.length;
+    for (let i = 0; i < maxLen; i++) {
+      const [row, col] = indexToCoordinates(i, paddedStrides);
+      const topRow = 0;
+      const bottomRow = imgRow - 1;
+      const leftCol = 0;
+      const rightCol = imgCol - 1;
+      const currentCol = col - pad;
+      const currentRow = row - pad;
+      if (row < pad) { // case: top
+        if (col < pad) { // case: top left
+          paddedMat[i] = img[coordinatesToIndex(topRow, leftCol, imagesStrides)]; // top left from original img
+        }
+        else if (col > imgCol + pad - 1) { // case: top right
+          paddedMat[i] = img[coordinatesToIndex(topRow, rightCol, imagesStrides)]; // top right from original img
+        }
+        else { // case: top, not corner
+          paddedMat[i] = img[coordinatesToIndex(topRow, currentCol, imagesStrides)]; // top from original img
+        }
+      }
+      else if (row > imgRow + pad - 1) { // case: bottom
+        if (col < pad) { // case: bottom left
+          paddedMat[i] = img[coordinatesToIndex(bottomRow, leftCol, imagesStrides)]; // bottom left from original img
+        }
+        else if (col > imgCol + pad - 1) { // case: bottom right
+          paddedMat[i] = img[coordinatesToIndex(bottomRow, rightCol, imagesStrides)]; // bottom right from original img
+        }
+        else { // case: bottom, not corner
+          paddedMat[i] = img[coordinatesToIndex(bottomRow, currentCol, imagesStrides)]; // bottom of original image
+        }
+      }
+      else if (col < pad) { // case: left
+        paddedMat[i] = img[coordinatesToIndex(currentRow, leftCol, imagesStrides)]; // left of original image
+      }
+      else if (col > imgCol + pad -1) { // case: right
+        paddedMat[i] = img[coordinatesToIndex(currentRow, rightCol, imagesStrides)]; // right of original image
+      }
+      else { // case: center
+        paddedMat[i] = img[coordinatesToIndex(currentRow, currentCol, imagesStrides)]; // original image value
+      }
+    }
+    return paddedMat;
+  }
+
+
+  const result = padInputImage(xValues)
+  console.log(`result ${result}`)
+
+  // Interpolation kernel
+  function interpolationKernel(x: number, a: number): number {
+    if (Math.abs(x) >= 0 && Math.abs(x) <= 1) {
+      return (a + 2) * (Math.abs(x) ** 3) - (a + 3) * (Math.abs(x) ** 2) + 1;
+    }
+    else if (Math.abs(x) > 1 && Math.abs(x) <= 2) {
+      return a * (Math.abs(x)**3)-(5*a)*(Math.abs(x)**2)+(8*a)*Math.abs(x)-4*a;
+    }
+    return 0;
+  }
+
+  console.log(`interpolationKernel ${interpolationKernel(1,2)}`)
+
+
+  return backend.makeTensorInfo(
+    [batch, newHeight, newWidth, numChannels], 'float32', result);
+
+}
+
+export const resizeBicubicConfig: KernelConfig = {
+  kernelName: ResizeBicubic,
+  backendName: 'cpu',
+  kernelFunc: resizeBicubic as unknown as KernelFunc
+};
+

tfjs-backend-cpu/src/register_all_kernels.ts

@@ -145,6 +145,9 @@ import {resizeBilinearConfig} from './kernels/ResizeBilinear';
 import {resizeBilinearGradConfig} from './kernels/ResizeBilinearGrad';
 import {resizeNearestNeighborConfig} from './kernels/ResizeNearestNeighbor';
 import {resizeNearestNeighborGradConfig} from './kernels/ResizeNearestNeighborGrad';
+import {resizeBicubicConfig} from './kernels/ResizeBicubic';
+// import {resizeBicubicGradConfig} from './kernels/ResizeBicubic';
+
 import {reverseConfig} from './kernels/Reverse';
 import {rotateWithOffsetConfig} from './kernels/RotateWithOffset';
 import {roundConfig} from './kernels/Round';
@@ -314,6 +317,7 @@ const kernelConfigs: KernelConfig[] = [
   reshapeConfig,
   resizeBilinearConfig,
   resizeBilinearGradConfig,
+  resizeBicubicConfig,
   resizeNearestNeighborConfig,
   resizeNearestNeighborGradConfig,
   reverseConfig,

tfjs-converter/python/BUILD

@@ -1,4 +1,5 @@
 load("@rules_python//python:packaging.bzl", "py_package", "py_wheel")
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -9,6 +10,23 @@ CONSOLE_SCRIPTS = {
     "tensorflowjs_wizard": "tensorflowjs.converters.wizard:pip_main",
 }
 
+compile_pip_requirements(
+    name = "tensorflowjs_deps_requirements",
+    extra_args = ["--allow-unsafe"],  # Allow pinning setuptools
+    requirements_in = "requirements.txt",
+    requirements_txt = "requirements_lock.txt",
+)
+
+compile_pip_requirements(
+    name = "tensorflowjs_dev_deps_requirements",
+    data = [
+        ":requirements.txt",
+    ],
+    extra_args = ["--allow-unsafe"],  # Allow pinning setuptools
+    requirements_in = "requirements-dev.txt",
+    requirements_txt = "requirements-dev_lock.txt",
+)
+
 py_package(
     name = "tensorflowjs_pkg",
     # Only include these Python packages.
@@ -48,7 +66,7 @@ py_wheel(
     license = "Apache 2.0",
     python_tag = "py3",
     requires = [
-        "flax>=0.5.3",
+        "flax>=0.6.2",
         "importlib_resources>=5.9.0",
         "jax>=0.3.16",
         "protobuf<3.20,>=3.9.2",

tfjs-converter/python/requirements-dev.txt

@@ -1,3 +1,4 @@
 -r requirements.txt
 PyInquirer==1.0.3
 pylint==2.5.0; python_version > '3.0'
+setuptools==65.6.3