Merge pull request #189 from City-of-Helsinki/release/2.4.0

Release/2.4.0

.github/workflows/production.yml

@@ -15,6 +15,11 @@ env:
   SERVICE_ARTIFACT_FOLDER: 'service_artifacts'
   APP_MIGRATE_COMMAND: /app/.prod/on_deploy.sh
   SERVICE_PORT: "8080"
+  K8S_REQUEST_CPU: 100m
+  K8S_REQUEST_RAM: 200Mi
+  K8S_LIMIT_CPU: 800m
+  K8S_LIMIT_RAM: 1Gi
+  K8S_REPLICACOUNT: 2
 
 jobs:
   build:
@@ -47,7 +52,7 @@ jobs:
           K8S_SECRET_SECRET_KEY: ${{ secrets.GH_STABLE_DJANGO_SECRET_KEY }}
           K8S_SECRET_VERSION: ${{ github.sha }}
           K8S_SECRET_ADMIN_USER_PASSWORD: ${{ secrets.STABLE_ADMIN_USER_PASSWORD }}
-          K8S_SECRET_HKM_FEEDBACK_FROM_EMAIL: "no-reply@hel.fi"
+          K8S_SECRET_HKM_DEFAULT_FROM_EMAIL: "no-reply@hel.fi"
           K8S_SECRET_HKM_FEEDBACK_NOTIFICATION_EMAILS: ${{ secrets.STABLE_HKM_FEEDBACK_NOTIFICATION_EMAILS }}
           K8S_SECRET_EMAIL_BACKEND: "anymail.backends.mailgun.EmailBackend"
           K8S_SECRET_MAIL_MAILGUN_KEY: ${{ secrets.GH_STABLE_MAILGUN_API_KEY }}

.github/workflows/review.yml

@@ -15,6 +15,10 @@ env:
   DATABASE_PASSWORD: testing-password
   APP_MIGRATE_COMMAND: /app/.prod/on_deploy.sh
   SERVICE_PORT: "8080"
+  K8S_REQUEST_CPU: 100m
+  K8S_REQUEST_RAM: 200Mi
+  K8S_LIMIT_CPU: 800m
+  K8S_LIMIT_RAM: 1Gi
 
 jobs:
   build:
@@ -47,7 +51,7 @@ jobs:
           K8S_SECRET_ALLOWED_HOSTS: "*"
           K8S_SECRET_DEBUG: 1
           K8S_SECRET_VERSION: ${{ github.sha }}
-          K8S_SECRET_HKM_FEEDBACK_FROM_EMAIL: "no-reply@hel.ninja"
+          K8S_SECRET_HKM_DEFAULT_FROM_EMAIL: "no-reply@hel.ninja"
           K8S_SECRET_ADMIN_USER_PASSWORD: ${{ secrets.STAGING_ADMIN_USER_PASSWORD }}
           K8S_SECRET_HKM_FEEDBACK_NOTIFICATION_EMAILS: "dummy-address@hel.ninja"
           K8S_SECRET_EMAIL_BACKEND: "anymail.backends.mailgun.EmailBackend"

.github/workflows/staging.yml

@@ -15,6 +15,11 @@ env:
   SERVICE_ARTIFACT_FOLDER: 'service_artifacts'
   APP_MIGRATE_COMMAND: /app/.prod/on_deploy.sh
   SERVICE_PORT: "8080"
+  K8S_REQUEST_CPU: 100m
+  K8S_REQUEST_RAM: 200Mi
+  K8S_LIMIT_CPU: 800m
+  K8S_LIMIT_RAM: 1Gi
+  K8S_REPLICACOUNT: 2
 
 jobs:
   build:
@@ -46,7 +51,7 @@ jobs:
           K8S_SECRET_SECRET_KEY: ${{ secrets.GH_QA_DJANGO_SECRET_KEY }}
           K8S_SECRET_ADMIN_USER_PASSWORD: ${{ secrets.STAGING_ADMIN_USER_PASSWORD }}
           K8S_SECRET_VERSION: ${{ github.sha }}
-          K8S_SECRET_HKM_FEEDBACK_FROM_EMAIL: "no-reply@hel.ninja"
+          K8S_SECRET_HKM_DEFAULT_FROM_EMAIL: "no-reply@hel.ninja"
           K8S_SECRET_HKM_FEEDBACK_NOTIFICATION_EMAILS: ${{ secrets.STAGING_HKM_FEEDBACK_NOTIFICATION_EMAILS }}
           K8S_SECRET_EMAIL_BACKEND: "anymail.backends.mailgun.EmailBackend"
           K8S_SECRET_MAIL_MAILGUN_KEY: ${{ secrets.GH_STAGING_MAILGUN_API_KEY }}
@@ -71,3 +76,17 @@ jobs:
           K8S_SECRET_DATABASE_DB: ${{ secrets.K8S_SECRET_DATABASE_DB_STAGING }}
           K8S_SECRET_DATABASE_USERNAME: ${{ secrets.K8S_SECRET_DATABASE_USERNAME_STAGING }}
           K8S_SECRET_DATABASE_PASSWORD: ${{ secrets.K8S_SECRET_DATABASE_PASSWORD_STAGING }}
+
+      - name: Deploy Cronjob
+        uses: City-of-Helsinki/setup-cronjob-action@main
+        with:
+          image_repository: ghcr.io/city-of-helsinki/${{ github.event.repository.name }}
+          image_tag:  ${{ github.sha }}
+          secret_name: project-staging-kuvaselaamo-secret
+          file_secret_name: project-staging-kuvaselaamo-file-secret
+          file_secret_mount_path: /tmp/secrets
+          kubeconfig_raw: ${{ env.KUBECONFIG_RAW }}
+          target_namespace: ${{ secrets.K8S_NAMESPACE_STAGING }}
+          schedule: '0 0 * * *'
+          command: "{/bin/sh}"
+          args: "{-c,cd /app && python manage.py clean_unused_data --days 365 && python manage.py clearsessions}"

.prod/uwsgi.ini

@@ -8,3 +8,6 @@ gid = nogroup
 master = 1
 processes = 2
 threads = 2
+; don't log readiness and healthz endpoints
+route = ^/readiness$ donotlog:
+route = ^/healthz$ donotlog:

CHANGELOG.md

@@ -3,6 +3,24 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
+# [2.4.0] 2021-03-17
+### Added
+- Added script that deletes old and unused data.
+- Added kubernetes cron job that executes clean_unused_data script and clearsessions (currently active only in 
+  staging environment).
+- Request a new password functionality.
+
+### Fixed
+- Added lang attribute to all collections. It will force screen reader language to finnish.
+- Fixed missing title tags to all unique pages.
+- If image is not found with image_id, correct error is shown.
+- Shopping cart buttons (increase, decrease, delete)
+- The shopping basket view now displays images based on the crop selection made by the user.
+
+### Changed
+- Moved CGS / Azure dependencies to environment specific requirements.in files.
+- Disabled logging for /healthz and /readiness endpoints.
+
 # [2.3.1] 2021-02-01
 ### Added
 - Temporary message stating that feedback processing is congested.

Dockerfile

@@ -12,8 +12,6 @@ RUN apt-install.sh \
     libpq-dev \
     && pip install --no-cache-dir \
     -r /app/requirements.txt \
-    && pip install --no-cache-dir \
-    -r /app/requirements-prod.txt \
     && apt-cleanup.sh \
     build-essential \
     pkg-config
@@ -34,7 +32,8 @@ FROM appbase as development
 # ==============================
 
 COPY --chown=appuser:appuser requirements-dev.txt /app/requirements-dev.txt
-RUN pip install --no-cache-dir -r /app/requirements-dev.txt \
+RUN pip install --no-cache-dir \
+    -r /app/requirements-dev.txt \
     && pip install --no-cache-dir pip-tools
 
 ENV DEV_SERVER=1
@@ -50,6 +49,17 @@ FROM appbase as production
 # ==============================
 
 COPY --from=staticbuilder --chown=appuser:appuser /app/static /app/static
+
+COPY --chown=appuser:appuser requirements-prod.txt /app/requirements-prod.txt
+RUN apt-install.sh \
+    build-essential \
+    libpq-dev \
+    && pip install --no-cache-dir \
+    -r /app/requirements-prod.txt \
+    && apt-cleanup.sh \
+    build-essential \
+    pkg-config
+
 COPY --chown=appuser:appuser . /app/
 
 USER appuser

README.md

@@ -122,6 +122,6 @@ Production: https://helsinkikuvia.fi
 
 ## CI/CD builds
 
-Project is using [Gitlab](https://gitlab.com/City-of-Helsinki/KuVa/github-mirrors/kuvaselaamo/pipelines)
+Project is using [GitHub Actions](https://github.com/City-of-Helsinki/kuvaselaamo/actions)
 for automated builds and deployment into the test environment.
 The test environment is built automatically from the `develop` branch.

docker-compose.env.example.yaml

@@ -45,3 +45,6 @@ MAIL_MAILGUN_API=
 
 # Level of logging. Defaults to ERROR in settings.py
 LOG_LEVEL=DEBUG
+
+# How many days will the password reset link be valid. Defaults to 1 in settings.
+PASSWORD_RESET_TIMEOUT_DAYS=1

hkm/email.py

@@ -27,7 +27,7 @@ def send_feedback_notification(feedback_id, force=False):
             count_sent_message = send_mail(
                 title,
                 message,
-                settings.HKM_FEEDBACK_FROM_EMAIL,
+                settings.DEFAULT_FROM_EMAIL,
                 settings.HKM_FEEDBACK_NOTIFICATION_EMAILS,
                 fail_silently=False,
             )

hkm/finna.py

@@ -112,13 +112,8 @@ def get_record(self, record_id):
                   'data': {'result_data': repr(result_data)}})
         return result_data
 
-    def get_image_url(self, record_id, w=0, h=0):
-        if w != 0 and h != 0:
-            url = 'https://finna.fi/Cover/Show?id=%s&w=%d&h=%d' % (
-                record_id, w, h)
-        else:
-            url = 'https://finna.fi/Cover/Show?id=%s&fullres=1&index=0' % record_id
-        return url
+    def get_image_url(self, record_id):
+        return 'https://finna.fi/Cover/Show?id=%s&fullres=1&index=0' % record_id
 
     def get_full_res_image_url(self, record_id):
         return 'https://finna.fi/Cover/Show?id=%s&size=master&index=0' % record_id

hkm/management/commands/clean_unused_data.py

@@ -0,0 +1,35 @@
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User
+from hkm.models.models import TmpImage, Feedback, ProductOrder
+from datetime import timedelta
+from django.utils import timezone
+
+DEFAULT_DAYS = 365
+
+class Command(BaseCommand):
+    help = "Clean unused data from application."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-d", "--days", type=int, help="Specifies the time in days for which old data is deleted", default=DEFAULT_DAYS
+        )
+
+    def handle(self, *args, **kwargs):
+        days = kwargs['days']
+        if not days:
+            days = DEFAULT_DAYS
+
+        self.stdout.write("Old data cleaning started! Removing data older than " + str(days) + " days.")
+        counted_date = timezone.now() - timedelta(days=days)
+
+        # Find and delete unused users. Do not delete superusers or staff users.
+        users = User.objects.filter(last_login__lte=counted_date, is_superuser=False, is_staff=False,
+                                    profile__is_museum=False, profile__is_admin=False).delete()
+
+        # Find and delete old TmpImages, Feedbacks and Orders
+        temps = TmpImage.delete_old_data(counted_date)
+        feedbacks = Feedback.delete_old_data(counted_date)
+        orders = ProductOrder.delete_old_data(counted_date)
+
+        self.stdout.write(self.style.SUCCESS('Old data cleaning finished! Removed %s user(s), %s temp(s), '
+                                             '%s feedback(s), %s order(s)' % (users[0], temps[0], feedbacks[0], orders[0])))

hkm/models/models.py

@@ -423,6 +423,10 @@ def save(self, *args, **kwargs):
             self.order_hash = get_random_string(20)
         return super(ProductOrder, self).save(*args, **kwargs)
 
+    @classmethod
+    def delete_old_data(self, date):
+        return self.objects.filter(user__isnull=True, modified__lte=date).delete()
+
     def __unicode__(self):
         return self.order_hash
 
@@ -441,6 +445,10 @@ class Feedback(BaseModel):
         verbose_name=_(u'Notification sent'), default=False)
     sent_from = models.CharField(verbose_name=_(u"Sent from"), max_length=500, null=True, blank=True)
 
+    @classmethod
+    def delete_old_data(self, date):
+        return self.objects.filter(user__isnull=True, modified__lte=date).delete()
+
 
 def get_tmp_upload_path(instance, filename):
     return 'tmp/%s' % filename
@@ -455,6 +463,10 @@ class TmpImage(BaseModel):
     edited_image = models.ImageField(verbose_name=_(
         u'Edited image'), upload_to=get_tmp_upload_path)
 
+    @classmethod
+    def delete_old_data(self, date):
+        return self.objects.filter(modified__lte=date, creator__isnull=True).delete()
+
     def __unicode__(self):
         return self.record_title
 

hkm/static/hkm/css/main.css

@@ -6360,35 +6360,50 @@ span.no-postal-fee {
 
 /* Login */
 .login-modal__dialog {
-  margin: -179px auto auto -149px;
+  margin: -179px 0 0 0;
   position: absolute;
   top: 50%;
-  left: 50%; }
+  width: 100%; }
 
 .login-modal__content {
   border-radius: 0; }
 
-.login-modal__signup-link {
-  font-family: "brownbold", sans-serif;
-  color: #005cab; }
-
 .login-modal__body {
-  padding: 15 25px; }
-  .login-modal__body .login-modal__btn {
-    margin: 0 0 15px 0; }
-  .login-modal__body .btn {
-    width: 100%; }
-
-@media (max-width: 768px) {
-  .login-modal__dialog {
-    margin: -179px auto auto -198px; } }
+  padding: 15px 75px; }
+  .login-modal__body_title {
+    color: #5268C2;
+    font: 18px "Signe", sans-serif;
+    margin-bottom: 25px; }
+  .login-modal__body_description {
+    font: 16px "brownlight", sans-serif;
+    margin-bottom: 25px; }
+  .login-modal__body_back-to, .login-modal__body_link {
+    color: #5268C2;
+    margin-top: 20px;
+    display: block;
+    font-family: "brownbold", sans-serif; }
+  .login-modal__body_svg {
+    margin-top: -3px;
+    height: 20px;
+    width: 20px; }
+  .login-modal__body_back-to {
+    color: #000; }
+  .login-modal__body_hr {
+    margin-top: 10px; }
+  .login-modal__body_btn {
+    min-width: 100%;
+    margin-bottom: 10px; }
+  .login-modal__body_error {
+    margin-top: 10px;
+    color: red; }
 
-@media (max-width: 400px) {
+@media (min-width: 500px) {
   .login-modal__dialog {
-    left: 0;
-    margin: -179px 15px auto 15px; }
-  .login-modal {
-    width: 100%; } }
+    margin: -179px auto auto -198px;
+    left: 50%;
+    width: 450px; }
+  .login-modal__body {
+    padding: 15px 100px; } }
 
 /* Main */
 .home-page {
@@ -7741,6 +7756,32 @@ a.welcome-modal__clarification {
   margin: 20px 0 40px;
   font-size: 16px; }
 
+/* GENERAL NEW STYLES 2021 */
+.hkm-button {
+  border-radius: 0;
+  color: #fff;
+  padding: 10px;
+  outline: 0;
+  font-family: "brownlight", sans-serif;
+  font-size: 16px;
+  font-weight: bold;
+  display: block;
+  text-align: center; }
+  .hkm-button__filled {
+    border: 0;
+    background-color: #935379;
+    color: #fff; }
+    .hkm-button__filled:hover {
+      color: #fff;
+      text-decoration: none; }
+  .hkm-button__outlined {
+    border: 1px solid #935379;
+    background-color: #fff;
+    color: #935379; }
+    .hkm-button__outlined:hover {
+      color: #935379;
+      text-decoration: none; }
+
 .navigation {
   height: 60px;
   background-color: #fff;