First steps to pypi package

.github/workflows/run_tests.yaml

@@ -21,7 +21,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           pip install --upgrade "jax[cpu]==0.4.10"
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          pip install .
 #      - name: Lint with flake8
 #        run: |
 #          # stop the build if there are Python syntax errors or undefined names

pyproject.toml

@@ -1,6 +1,61 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "levanter"
+version = "0.0.1"
+authors = [
+  { name="David Hall", email="dlwh@cs.stanford.edu" },
+  {name="Ivan Zhou", email="ivanz@stanford.edu"}
+]
+description = "Scalable Training for Foundation Models with Named Tensors and Jax"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License 2.0",
+    "Operating System :: Linux",
+    "Operating System :: MacOS :: MacOS X",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+]
+
+dependencies = [
+    # we'll require that you install jax yourself, since the extras vary by system.
+    # jax = {version = ">=0.4.10,<0.5.0"}
+    "equinox~=0.9.0",
+    "transformers>=4.22.0",
+    "optax",
+    "wandb",
+    "pyrallis>=0.3.1",
+    "pyarrow>=11.0.0",
+    "zstandard>=0.20.0",
+    "datasets==2.11.0",
+    "gcsfs==2023.3.0",
+    "braceexpand>=0.1.7",
+    "jmp>=0.0.3",
+    "fsspec==2023.3.0", # pin this to make gcsfs happy
+    "tensorstore>=0.1.36",
+    "pytimeparse>=1.1.8",
+    "humanfriendly==10.0",
+    "requests",
+    "safetensors[numpy]",
+    "matplotlib>=3.7.0",
+    "tblib>=1.7.0,<2.0.0",
+    "dataclasses_json",
+    "ray[default]",
+    "rich>=13",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/stanford-crfm/levanter"
+"Bug Tracker" = "https://github.com/stanford-crfm/levanter/issues"
+
+
 [tool.black]
 line-length = 119
-target-version = ["py36", "py37", "py38"]
+target-version = ["py310"]
 experimental_string_processing = true
 
 [tool.isort]

setup.py

@@ -1,12 +1,13 @@
+# get path to current directory
+import os
+
 import setuptools
 
 
-with open("README.md", "r", encoding="utf-8") as fh:
-    long_description = fh.read()
+curdir = os.path.dirname(os.path.abspath(__file__))
 
-# hacky, but https://stackoverflow.com/questions/14399534/reference-requirements-txt-for-the-install-requires-kwarg-in-setuptools-setup-py
-with open("requirements.txt", "r") as f:
-    install_reqs = [line for line in f.readlines() if not line.startswith("#")]
+with open(f"{curdir}/README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
 
 setuptools.setup(
     name="levanter",
@@ -15,8 +16,7 @@
     url="https://github.com/stanford-crfm/levanter",
     author="David Hall",
     author_email="dlwh@cs.stanford.edu",
-    license="Apache 2.0",
-    install_requires=install_reqs,
+    # install_requires=install_reqs,
     long_description=long_description,
     packages=setuptools.find_packages(where="src", exclude=("tests",)),
     # https://stackoverflow.com/questions/70777486/pip-install-e-doesnt-allow-to-import-via-package-dir