migrate from csv files to sqlite databases for downstream use in queries

.github/workflows/lint.yml

@@ -0,0 +1,39 @@
+name: Lint
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main, staging-* ]
+  workflow_dispatch: 
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: pip cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: lint-pip-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            lint-pip-
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+
+      - name: Lint
+        run: ./scripts/lint.sh

.github/workflows/test.yml

@@ -0,0 +1,59 @@
+name: Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main, staging-* ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10']
+        os: [ubuntu-latest]  # , macos-latest]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Ubuntu cache
+        uses: actions/cache@v1
+        if: startsWith(matrix.os, 'ubuntu')
+        with:
+          path: ~/.cache/pip
+          key:
+            ${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ matrix.os }}-${{ matrix.python-version }}-
+
+      - name: macOS cache
+        uses: actions/cache@v1
+        if: startsWith(matrix.os, 'macOS')
+        with:
+          path: ~/Library/Caches/pip
+          key:
+            ${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ matrix.os }}-${{ matrix.python-version }}-
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[dev]
+
+      - name: Generate databases
+        run: python cities/utils/csv_to_db_pipeline.py
+
+      - name: Test
+        run: python -m pytest tests/
+
+      - name: Test Notebooks
+        run: |
+            ./scripts/test_notebooks.sh

cities/__init__.py

@@ -2,4 +2,5 @@
 
 Project short description.
 """
+
 __version__ = "0.0.1"

cities/modeling/model_interactions.py

@@ -3,10 +3,10 @@
 from typing import Optional
 
 import dill
+import pyro.distributions as dist
 import torch
 
 import pyro
-import pyro.distributions as dist
 from cities.modeling.modeling_utils import (
     prep_wide_data_for_inference,
     train_interactions_model,
@@ -50,12 +50,12 @@ def __init__(
 
         self.model_args = self.data["model_args"]
 
-        self.model_conditioned = pyro.condition(
+        self.model_conditioned = pyro.condition(  # type: ignore
             self.model,
             data={"T": self.data["t"], "Y": self.data["y"], "X": self.data["x"]},
         )
 
-        self.model_rendering = pyro.render_model(
+        self.model_rendering = pyro.render_model(  # type: ignore
             self.model, model_args=self.model_args, render_distributions=True
         )
 

cities/modeling/modeling_utils.py

@@ -3,6 +3,9 @@
 import matplotlib.pyplot as plt
 import pandas as pd
 import torch
+from pyro.infer import SVI, Trace_ELBO
+from pyro.infer.autoguide import AutoNormal
+from pyro.optim import Adam  # type: ignore
 from scipy.stats import spearmanr
 
 import pyro
@@ -11,9 +14,6 @@
     list_available_features,
     list_tensed_features,
 )
-from pyro.infer import SVI, Trace_ELBO
-from pyro.infer.autoguide import AutoNormal
-from pyro.optim import Adam  # type: ignore
 
 
 def drop_high_correlation(df, threshold=0.85):
@@ -217,7 +217,7 @@ def train_interactions_model(
     lr: float = 0.01,
 ):
     guide = None
-    pyro.clear_param_store()
+    pyro.clear_param_store()  # type: ignore
 
     guide = AutoNormal(conditioned_model)
 

cities/queries/causal_insight_slim.py

@@ -500,10 +500,10 @@ def get_fips_predictions(
         difference = (
             self.predictions_original["observed"] - self.observed_outcomes_original
         )
-        self.predictions_original[
-            ["observed", "mean", "low", "high"]
-        ] = self.predictions_original[["observed", "mean", "low", "high"]].sub(
-            difference, axis=0
+        self.predictions_original[["observed", "mean", "low", "high"]] = (
+            self.predictions_original[["observed", "mean", "low", "high"]].sub(
+                difference, axis=0
+            )
         )
 
     def plot_predictions(

cities/queries/fips_query.py

@@ -467,9 +467,9 @@ def find_euclidean_kins(self):
                 if col.endswith(feature)
             ]
             if _selected:
-                atemporal_aggregated_dict[
-                    feature
-                ] = atemporal_featurewise_contributions_df[_selected].sum(axis=1)
+                atemporal_aggregated_dict[feature] = (
+                    atemporal_featurewise_contributions_df[_selected].sum(axis=1)
+                )
 
         aggregated_atemporal_featurewise_contributions_df = pd.DataFrame(
             atemporal_aggregated_dict
@@ -489,9 +489,9 @@ def find_euclidean_kins(self):
             axis=1,
         )
         columns_to_normalize = self.aggregated_featurewise_contributions.iloc[:, 3:]
-        self.aggregated_featurewise_contributions.iloc[
-            :, 3:
-        ] = columns_to_normalize.div(columns_to_normalize.sum(axis=1), axis=0)
+        self.aggregated_featurewise_contributions.iloc[:, 3:] = (
+            columns_to_normalize.div(columns_to_normalize.sum(axis=1), axis=0)
+        )
 
         # some sanity checks
         count = sum([1 for distance in distances if distance == 0])

cities/utils/clean_variable.py

@@ -1,7 +1,7 @@
 import numpy as np
 import pandas as pd
 
-from cities.utils.clean_gdp import clean_gdp
+from cities.utils.cleaning_scripts.clean_gdp import clean_gdp
 from cities.utils.cleaning_utils import standardize_and_scale
 from cities.utils.data_grabber import DataGrabber, find_repo_root
 

cities/utils/cleaning_pipeline.py

@@ -1,26 +1,38 @@
-from cities.utils.clean_age_composition import clean_age_composition
-from cities.utils.clean_burdens import clean_burdens
-from cities.utils.clean_ethnic_composition import clean_ethnic_composition
-from cities.utils.clean_ethnic_composition_ma import clean_ethnic_composition_ma
-from cities.utils.clean_gdp import clean_gdp
-from cities.utils.clean_gdp_ma import clean_gdp_ma
-from cities.utils.clean_hazard import clean_hazard
-from cities.utils.clean_homeownership import clean_homeownership
-from cities.utils.clean_income_distribution import clean_income_distribution
-from cities.utils.clean_industry import clean_industry
-from cities.utils.clean_industry_ma import clean_industry_ma
-from cities.utils.clean_industry_ts import clean_industry_ts
-from cities.utils.clean_population import clean_population
-from cities.utils.clean_population_density import clean_population_density
-from cities.utils.clean_population_ma import clean_population_ma
-from cities.utils.clean_spending_commerce import clean_spending_commerce
-from cities.utils.clean_spending_HHS import clean_spending_HHS
-from cities.utils.clean_spending_transportation import clean_spending_transportation
-from cities.utils.clean_transport import clean_transport
-from cities.utils.clean_unemployment import clean_unemployment
-from cities.utils.clean_urbanicity_ma import clean_urbanicity_ma
-from cities.utils.clean_urbanization import clean_urbanization
-from cities.utils.cleaning_poverty import clean_poverty
+from cities.utils.cleaning_scripts.clean_age_composition import clean_age_composition
+from cities.utils.cleaning_scripts.clean_burdens import clean_burdens
+from cities.utils.cleaning_scripts.clean_ethnic_composition import (
+    clean_ethnic_composition,
+)
+from cities.utils.cleaning_scripts.clean_ethnic_composition_ma import (
+    clean_ethnic_composition_ma,
+)
+from cities.utils.cleaning_scripts.clean_gdp import clean_gdp
+from cities.utils.cleaning_scripts.clean_gdp_ma import clean_gdp_ma
+from cities.utils.cleaning_scripts.clean_hazard import clean_hazard
+from cities.utils.cleaning_scripts.clean_homeownership import clean_homeownership
+from cities.utils.cleaning_scripts.clean_income_distribution import (
+    clean_income_distribution,
+)
+from cities.utils.cleaning_scripts.clean_industry import clean_industry
+from cities.utils.cleaning_scripts.clean_industry_ma import clean_industry_ma
+from cities.utils.cleaning_scripts.clean_industry_ts import clean_industry_ts
+from cities.utils.cleaning_scripts.clean_population import clean_population
+from cities.utils.cleaning_scripts.clean_population_density import (
+    clean_population_density,
+)
+from cities.utils.cleaning_scripts.clean_population_ma import clean_population_ma
+from cities.utils.cleaning_scripts.clean_spending_commerce import (
+    clean_spending_commerce,
+)
+from cities.utils.cleaning_scripts.clean_spending_HHS import clean_spending_HHS
+from cities.utils.cleaning_scripts.clean_spending_transportation import (
+    clean_spending_transportation,
+)
+from cities.utils.cleaning_scripts.clean_transport import clean_transport
+from cities.utils.cleaning_scripts.clean_unemployment import clean_unemployment
+from cities.utils.cleaning_scripts.clean_urbanicity_ma import clean_urbanicity_ma
+from cities.utils.cleaning_scripts.clean_urbanization import clean_urbanization
+from cities.utils.cleaning_scripts.cleaning_poverty import clean_poverty
 
 # from cities.utils.clean_health import clean_health