diff --git a/CHANGELOG.md b/CHANGELOG.md index fc4fb520..7625a79d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,59 @@ # Changelog +## v0.3.0a1 (2022-12-22) + + * [88d26ec](https://github.com/numaproj/numalogic/commit/88d26ec000ee719f967dc90f4f78f57784b1db7e) feat!: convert AE variants to lightning modules (#110) + * [ed94615](https://github.com/numaproj/numalogic/commit/ed9461517f68192d275d81ef38727bcbe62e887f) fix: fix and clean mlflow test cases (#109) + +### Contributors + + * Avik Basu + * Kushal Batra + +## v0.3.0a0 (2022-12-08) + + * [78cf5b4](https://github.com/numaproj/numalogic/commit/78cf5b4e26ffd6dffa5e0dc364cbd51b220bd928) fix: fix pipeline for 0.3 release (#106) + * [709553f](https://github.com/numaproj/numalogic/commit/709553f4ae879f7e786634b02867966756706174) fix: fix mlflow test cases (#98) + * [701812e](https://github.com/numaproj/numalogic/commit/701812efed6742915d9cf57aef985127b3cc449d) feat!: disentangle threshold selection from the main model (#89) + +### Contributors + + * Avik Basu + * Kushal Batra + +## v0.2.9 (2022-12-21) + + * [df20591](https://github.com/numaproj/numalogic/commit/df20591f5a3b45117525c4331e5668c4289bd118) fix: sklearn base import for scikit learn v1.2 (#112) + +### Contributors + + * Avik Basu + +## v0.2.8 (2022-11-29) + + * [7d5075d](https://github.com/numaproj/numalogic/commit/7d5075ddda5950fef772cf07ffdb8f949bed589f) remove mlflow full + * [05d6071](https://github.com/numaproj/numalogic/commit/05d6071a1de4ca85ed8095bab7654c2e77f366f5) fix: have mlflow-server as an optional extra + * [3c5bc83](https://github.com/numaproj/numalogic/commit/3c5bc8371a50a98a7ffc634b034692cc53ba597c) fix: lock file + * [d2c6293](https://github.com/numaproj/numalogic/commit/d2c6293f594ed67cc8a6ddabce1f8f0bdd84249f) fix: change example pipeline mlflow port (#96) + +### Contributors + + * Avik Basu + * Kushal Batra + +## v0.2.7 (2022-11-14) + + * [f9c74c9](https://github.com/numaproj/numalogic/commit/f9c74c90c37fca24db3465c24326a639e93bc802) fix: allow only patch updates in torch version due to cuda build errors on mac (#90) + * [0a5fcdf](https://github.com/numaproj/numalogic/commit/0a5fcdf6c1ea4732b2c6c97f57d983751cfe2956) fix_readme: mention namespace name when applying the pipeline (#88) + * [537fae5](https://github.com/numaproj/numalogic/commit/537fae56577954391d10b134cb318d388c1a7dd7) fix: AutoencoderPipeline logged loss mean (#55) + * [22f8e5d](https://github.com/numaproj/numalogic/commit/22f8e5d79183d848f68c7cbeb2d88126fb326d03) chore!: make mlflow as an optional dependency (#47) + +### Contributors + + * Avik Basu + * Kushal Batra + * diego-ponce + ## v0.2.6 (2022-10-17) * [5703d1b](https://github.com/numaproj/numalogic/commit/5703d1b75d642242dc5c521a2e85e9cbb2527923) fix: update readme with optional mlflow dependency diff --git a/Makefile b/Makefile index d5a2c003..31b0beed 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ lint: format # install all dependencies setup: - poetry install --all-extras + poetry install --with dev --all-extras # test your application (tests in the tests/ directory) test: diff --git a/docs/quick-start.md b/docs/quick-start.md index 5f207d3f..bb86c20d 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -83,18 +83,17 @@ Once Numaflow is installed, create a simple Numalogic pipeline, which takes in t For building this pipeline, navigate to [numalogic-simple-pipeline](https://github.com/numaproj/numalogic/tree/main/examples/numalogic-simple-pipeline) under the examples folder and execute the following commands. -1. Build the docker image, import it to k3d, and apply the pipeline. +1. Apply the pipeline. *Note Make sure the pipeline and, numaflow controllers and isbsvc pods are running in the same namespace (`default` in this case).* ```shell -docker build -t numalogic-simple-pipeline:v1 . && k3d image import docker.io/library/numalogic-simple-pipeline:v1 - kubectl apply -f numa-pl.yaml ``` 2. To verify if the pipeline has been deployed successfully, check the status of each pod. ```shell -> kubectl get pods +kubectl get pods +``` +Output will be something like this: +``` NAME READY STATUS RESTARTS AGE -numaflow-server-d64bf6f7c-2czd7 1/1 Running 0 72s -numaflow-controller-c84948cbb-994fn 1/1 Running 0 72s isbsvc-default-js-0 3/3 Running 0 68s isbsvc-default-js-1 3/3 Running 0 68s isbsvc-default-js-2 3/3 Running 0 68s @@ -111,9 +110,9 @@ numalogic-simple-pipeline-in-0-tmd0v 1/1 Running 0 Once the pipeline has been created, the data can be sent to the pipeline by port-forwarding the input vertex. -1. Port-forward to the http-source vertex +1. Port-forward to the http-source vertex. From the above pod output, this would be: ```shell - kubectl port-forward simple-numalogic-pipeline-in-0-xxxxx 8443 + kubectl port-forward numalogic-simple-pipeline-in-0-tmd0v 8443 ``` 2. Send the data to the pod via curl diff --git a/examples/numalogic-simple-pipeline/.dockerignore b/examples/numalogic-simple-pipeline/.dockerignore new file mode 100644 index 00000000..087f5822 --- /dev/null +++ b/examples/numalogic-simple-pipeline/.dockerignore @@ -0,0 +1,6 @@ +.git/ +__pycache__/ +**/__pycache__/ +*.py[cod] +*$py.class +.idea/ diff --git a/examples/numalogic-simple-pipeline/Dockerfile b/examples/numalogic-simple-pipeline/Dockerfile index df0a894e..d5fa292a 100644 --- a/examples/numalogic-simple-pipeline/Dockerfile +++ b/examples/numalogic-simple-pipeline/Dockerfile @@ -1,3 +1,7 @@ +#################################################################################################### +# builder: install needed dependencies +#################################################################################################### + FROM python:3.10-slim-bullseye AS builder ENV PYTHONFAULTHANDLER=1 \ @@ -27,9 +31,11 @@ RUN apt-get update \ # install dumb-init && wget -O /dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 \ && chmod +x /dumb-init # \ - # && curl -sSL https://install.python-poetry.org | python3 - && pip3 install --upgrade pip3 +#################################################################################################### +# mlflow: used for running the mlflow server +#################################################################################################### FROM builder AS mlflow WORKDIR $PYSETUP_PATH @@ -45,7 +51,9 @@ CMD ["/app/entry.sh"] EXPOSE 5000 - +#################################################################################################### +# udf: used for running the udf vertices +#################################################################################################### FROM builder AS udf WORKDIR $PYSETUP_PATH @@ -60,4 +68,4 @@ RUN chmod +x entry.sh ENTRYPOINT ["/dumb-init", "--"] CMD ["/app/entry.sh"] -EXPOSE 5000 \ No newline at end of file +EXPOSE 5000 diff --git a/examples/numalogic-simple-pipeline/numa-pl.yaml b/examples/numalogic-simple-pipeline/numa-pl.yaml index cd480a30..92f14b8c 100644 --- a/examples/numalogic-simple-pipeline/numa-pl.yaml +++ b/examples/numalogic-simple-pipeline/numa-pl.yaml @@ -18,7 +18,7 @@ spec: min: 1 udf: container: - image: docker.io/library/numalogic-simple-pipeline:v0.3.0a1 + image: quay.io/numaio/numalogic/example-udf env: - name: WIN_SIZE value: "12" @@ -31,7 +31,7 @@ spec: min: 1 udf: container: - image: docker.io/library/numalogic-simple-pipeline:v0.3.0a1 + image: quay.io/numaio/numalogic/example-udf env: - name: WIN_SIZE value: "12" @@ -44,7 +44,7 @@ spec: min: 1 udf: container: - image: docker.io/library/numalogic-simple-pipeline:v0.3.0a1 + image: quay.io/numaio/numalogic/example-udf env: - name: WIN_SIZE value: "12" @@ -57,7 +57,7 @@ spec: min: 1 udf: container: - image: docker.io/library/numalogic-simple-pipeline:v0.3.0a1 + image: quay.io/numaio/numalogic/example-udf env: - name: WIN_SIZE value: "12" @@ -78,7 +78,7 @@ spec: env: - name: WIN_SIZE value: "12" - image: docker.io/library/numalogic-simple-pipeline:v0.3.0a1 + image: quay.io/numaio/numalogic/example-udf args: - python - starter.py @@ -121,7 +121,7 @@ spec: app: mlflow spec: containers: - - image: docker.io/library/numalogic-simple-pipeline:mlflow-v0.3.0a1 + - image: quay.io/numaio/numalogic/example-mlflow name: mlflow args: - server diff --git a/poetry.lock b/poetry.lock index 00bda9f3..b1f38ede 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3979,9 +3979,9 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -mlflow-skinny = ["mlflow-skinny"] +mlflow = ["mlflow-skinny"] [metadata] lock-version = "2.0" python-versions = ">=3.8, <3.11" -content-hash = "84d8cdfba59fafcf6bbc6bb260f3916338b9b01563cea0f80538785eb587d079" +content-hash = "2877abf2a884cbf306f73f0eead4e733ac91c76de4d98808e42062f2e246058f" diff --git a/pyproject.toml b/pyproject.toml index b7456b19..dc79a1c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "numalogic" -version = "0.3.0a1" +version = "0.3.0" description = "Collection of operational Machine Learning models and tools." authors = ["Numalogic Developers"] packages = [{ include = "numalogic" }] @@ -28,7 +28,7 @@ scikit-learn = "^1.0" mlflow-skinny = { version = "~2.0.1", optional = true } [tool.poetry.extras] -mlflow-skinny = ["mlflow-skinny"] +mlflow = ["mlflow-skinny"] [tool.poetry.group.torch] optional = true