Upgrading Sanic and yt-dlp. Major changes occurred because Sanic chan…

…ged how multiprocessing works. Python 3.11 is now required, old versions were causing unpredictability in tests. (Sanic does not yet support 3.12) Sanic has been upgraded to 23.6.0, which is the latest version that avoids this bug: sanic-org/sanic#2921 New strategy for multiprocessing is to create all multiprocessing tools in one process, then fork to other processes. The previous strategy was to declare multiprocessing tools at the top of every file, or wherever they were needed at import/creation. Now all multiprocessing tools are attached to the app.shared_ctx. This means `api_app` is imported in many, many places. This forced a change in how the DownloadManager works. Previously, it would continually run download workers which would pull downloads from a multiprocessing.Queue. Now, a single worker checks for new downloads and sends a Sanic signal. Flags have been reworked to use the `api_app`. I removed the `which` flag functionality because the `which` are called at import and needed their own multiprocessing.Event.
lrnselfreliance · Apr 30, 2024 · 07c6101 · 07c6101
1 parent 2c127a0
commit 07c6101
Show file tree

Hide file tree

Showing 65 changed files with 1,233 additions and 1,053 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -61,91 +61,10 @@ jobs:
           paths:
             - ./app/node_modules
       - run: cd app && npm run test
-  api-tests-3-8:
-    docker:
-      - image: cimg/python:3.8
-      - image: cimg/postgres:12.9
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_DB: wrolpi
-          POSTGRES_PASSWORD: "wrolpi"
-    resource_class: medium
-    steps:
-      - checkout
-      - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg catdoc
-      - restore_cache:
-          key: deps-3.8-{{ checksum "requirements.txt" }}-2
-      - run:
-          name: Install Requirements
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install -r requirements.txt
-      - save_cache:
-          key: deps-3.8-{{ checksum "requirements.txt" }}
-          paths:
-            - "venv"
-      - run:
-          command: './venv/bin/pytest -svv'
-  api-tests-3-9:
-    docker:
-      - image: cimg/python:3.9
-      - image: cimg/postgres:13.5
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_DB: wrolpi
-          POSTGRES_PASSWORD: "wrolpi"
-    resource_class: medium
-    steps:
-      - checkout
-      - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg catdoc
-      - restore_cache:
-          key: deps-3.9-{{ checksum "requirements.txt" }}-2
-      - run:
-          name: Install Requirements
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install -r requirements.txt
-      - save_cache:
-          key: deps-3.9-{{ checksum "requirements.txt" }}
-          paths:
-            - "venv"
-      - run:
-          command: './venv/bin/pytest -svv'
-  api-tests-3-10:
-    docker:
-      - image: cimg/python:3.10
-      - image: cimg/postgres:14.1
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_DB: wrolpi
-          POSTGRES_PASSWORD: "wrolpi"
-    resource_class: medium
-    steps:
-      - checkout
-      - run: sudo apt-get update
-      - run: sudo apt-get install -y ffmpeg catdoc
-      - restore_cache:
-          key: deps-3.10-{{ checksum "requirements.txt" }}-2
-      - run:
-          name: Install Requirements
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install -r requirements.txt
-      - save_cache:
-          key: deps-3.10-{{ checksum "requirements.txt" }}
-          paths:
-            - "venv"
-      - run:
-          command: './venv/bin/pytest -svv'
   api-tests-3-11:
     docker:
       - image: cimg/python:3.11
-      - image: cimg/postgres:14.1
+      - image: cimg/postgres:15.6
         environment:
           POSTGRES_USER: postgres
           POSTGRES_DB: wrolpi
@@ -172,7 +91,7 @@ jobs:
   api-tests-3-12:
     docker:
       - image: cimg/python:3.12
-      - image: cimg/postgres:14.1
+      - image: cimg/postgres:15.6
         environment:
           POSTGRES_USER: postgres
           POSTGRES_DB: wrolpi
@@ -200,10 +119,8 @@ jobs:
 workflows:
   wrolpi-api-tests:
     jobs:
-      - api-tests-3-8
-      - api-tests-3-9
-      - api-tests-3-10
       - api-tests-3-11
+#      - api-tests-3-12 Sanic does not yet support 3.12.
   wrolpi-app-test:
     jobs:
       - app-tests-14

diff --git a/.gitignore b/.gitignore
@@ -126,6 +126,7 @@ docker-compose.override.yml
 
 # test directory is used as media directory, we don't want to commit what a user downloads.
 test
+pg_data
 
 # Directories used to build images
 /debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz

diff --git a/app/src/api.js b/app/src/api.js
@@ -772,6 +772,20 @@ export async function clearFailedDownloads() {
     }
 }
 
+export async function deleteOnceDownloads() {
+    let response = await apiPost(`${API_URI}/download/delete_once`);
+    if (response.status === 204) {
+        return null
+    } else {
+        toast({
+            type: 'error',
+            title: 'Error!',
+            description: 'Could not delete once downloads!  See server logs.',
+            time: 5000,
+        });
+    }
+}
+
 export async function getStatistics() {
     let response = await apiGet(`${API_URI}/statistics`);
     if (response.status === 200) {

diff --git a/app/src/components/Common.js b/app/src/components/Common.js
@@ -1583,6 +1583,15 @@ export function InfoMessage({children, size = null}) {
     </Message>
 }
 
+export function HandPointMessage({children, size = null}) {
+    return <Message info icon size={size}>
+        <SIcon name='hand point right'/>
+        <Message.Content>
+            {children}
+        </Message.Content>
+    </Message>
+}
+
 export function WarningMessage({children, size = null}) {
     return <Message warning icon size={size}>
         <SIcon name='hand point right'/>

diff --git a/app/src/components/Map.js b/app/src/components/Map.js
@@ -2,6 +2,7 @@ import React, {useContext} from "react";
 import {
     APIButton,
     ErrorMessage,
+    HandPointMessage,
     HelpPopup,
     humanFileSize,
     IframeViewer,
@@ -31,6 +32,9 @@ import {Loader, Placeholder, Table} from "./Theme";
 import {StatusContext} from "../contexts/contexts";
 import _ from "lodash";
 
+
+const VIEWER_URL = `http://${window.location.hostname}:8084/`;
+
 function DockerMapImportWarning() {
     const {status} = useContext(StatusContext);
     if (status['dockerized']) {
@@ -89,6 +93,12 @@ function DownloadMessage() {
     </InfoMessage>
 }
 
+const ViewerMessage = () => {
+    return <HandPointMessage>
+        <p>You can view your Map at <a href={VIEWER_URL}>{VIEWER_URL}</a></p>
+    </HandPointMessage>
+}
+
 function SlowImportMessage() {
     const {status} = useContext(StatusContext);
     if (status && status['cpu_info'] && status['cpu_info']['temperature'] >= 80) {
@@ -277,12 +287,13 @@ class ManageMap extends React.Component {
                 </TableFooter>
             </Table>
             <DownloadMessage/>
+            <ViewerMessage/>
         </PageContainer>
     }
 }
 
 function MapPage() {
-    return <IframeViewer title='map' viewerUrl={`http://${window.location.hostname}:8084/`}/>
+    return <IframeViewer title='map' viewerUrl={VIEWER_URL}/>
 }
 
 export function MapRoute() {

diff --git a/app/src/components/Zim.js b/app/src/components/Zim.js
@@ -33,6 +33,7 @@ import {
     APIButton,
     encodeMediaPath,
     ErrorMessage,
+    HandPointMessage,
     humanFileSize,
     IframeViewer,
     InfoMessage,
@@ -328,14 +329,18 @@ export const ZimSearchView = ({suggestions, loading}) => {
     </>
 }
 
-const ViewerMessage = () => {
+const DownloadMessage = () => {
     return <InfoMessage>
         <p>More Zim files are available from the full Kiwix library&nbsp;
             <a href='https://download.kiwix.org/'>https://download.kiwix.org/</a>
         </p>
+    </InfoMessage>
+}
 
+const ViewerMessage = () => {
+    return <HandPointMessage>
         <p>You can view your Zim files using the Kiwix app, or at <a href={VIEWER_URL}>{VIEWER_URL}</a></p>
-    </InfoMessage>
+    </HandPointMessage>
 }
 
 const ZimCatalogItemRow = ({item, subscriptions, iso_639_codes, fetchSubscriptions}) => {
@@ -524,6 +529,7 @@ class ManageZim extends React.Component {
 
             <Divider/>
 
+            <DownloadMessage/>
             <ViewerMessage/>
         </PageContainer>
     }

diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js
@@ -1,5 +1,12 @@
 import React from "react";
-import {clearCompletedDownloads, clearFailedDownloads, deleteDownload, killDownload, restartDownload} from "../../api";
+import {
+    clearCompletedDownloads,
+    clearFailedDownloads,
+    deleteDownload,
+    deleteOnceDownloads,
+    killDownload,
+    restartDownload
+} from "../../api";
 import {Link} from "react-router-dom";
 import {
     APIButton,
@@ -40,7 +47,7 @@ function ClearCompleteDownloads({callback}) {
     return <>
         <APIButton
             onClick={localClearDownloads}
-            color='yellow'
+            color='violet'
             obeyWROLMode={true}
         >Clear Completed</APIButton>
     </>
@@ -58,7 +65,7 @@ function ClearFailedDownloads({callback}) {
     }
 
     return <APIButton
-        color='red'
+        color='yellow'
         onClick={localDeleteFailed}
         confirmContent='Are you sure you want to delete failed downloads?  They will not be retried.'
         confirmButton='Delete'
@@ -68,6 +75,28 @@ function ClearFailedDownloads({callback}) {
     </APIButton>
 }
 
+function DeleteOnceDownloads({callback}) {
+    async function localDeleteOnce() {
+        try {
+            await deleteOnceDownloads();
+        } finally {
+            if (callback) {
+                callback()
+            }
+        }
+    }
+
+    return <APIButton
+        color='red'
+        onClick={localDeleteOnce}
+        confirmContent='Are you sure you want to delete all downloads?  They may be retried!'
+        confirmButton='Delete'
+        obeyWROLMode={true}
+    >
+        Delete All
+    </APIButton>
+}
+
 class RecurringDownloadRow extends React.Component {
     constructor(props) {
         super(props);
@@ -306,6 +335,7 @@ export function OnceDownloadsTable({downloads, fetchDownloads}) {
                         <TableHeaderCell colSpan={3}>
                             <ClearCompleteDownloads callback={fetchDownloads}/>
                             <ClearFailedDownloads callback={fetchDownloads}/>
+                            <DeleteOnceDownloads callback={fetchDownloads}/>
                         </TableHeaderCell>
                     </TableRow>
                 </TableFooter>

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -11,6 +11,8 @@ services:
     healthcheck:
       test: [ 'CMD-SHELL', 'pg_isready -U postgres' ]
       interval: 10s
+    volumes:
+      - ./pg_data:/var/lib/postgresql/data
 
   api:
     depends_on:
@@ -29,7 +31,6 @@ services:
       - './alembic.ini:/opt/wrolpi/alembic.ini'
     ports:
       - ${REACT_APP_API-127.0.0.1:8081}:8081
-    command: '-vv api --host 0.0.0.0'
     user: '${UID-1000}:${GID-1000}'
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]

diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
@@ -8,12 +8,14 @@ RUN apt update
 RUN apt-get install -y ffmpeg catdoc
 RUN ffmpeg -version
 
-# Install WROLPi
+# Install dependencies.
+COPY requirements.txt /opt/wrolpi/requirements.txt
+RUN pip3 install -r /opt/wrolpi/requirements.txt
+
+# Install WROLPi.
 COPY main.py /opt/wrolpi/
 COPY wrolpi /opt/wrolpi/wrolpi
 COPY modules /opt/wrolpi/modules
-COPY requirements.txt /opt/wrolpi/requirements.txt
-RUN pip3 install -r /opt/wrolpi/requirements.txt
 
 ENTRYPOINT [ "python3", "-OO", "/opt/wrolpi/main.py"]
-CMD ["api", "--host", "0.0.0.0" ]
+CMD ["-vv", "api", "--host", "0.0.0.0"]