Port utils::convert_text() and UTF-8 <-> locale converters to Rust

.cirrus.yml

@@ -47,7 +47,7 @@ freebsd_task:
     # possible before failing. This enables developers to fix more errors before
     # re-submitting the code to CI, which should increase throughput.
     build_script: su testuser -c 'cd ~ && gmake --jobs=5 --keep-going all test'
-    test_script: su testuser -c 'cd ~ && RUST_TEST_THREADS=5 RUST_BACKTRACE=1 gmake -j5 ci-check'
+    test_script: su testuser -c 'cd ~ && RUST_TEST_THREADS=5 RUST_BACKTRACE=1 gmake -j5 NEWSBOAT_RUN_IGNORED_TESTS=1 ci-check'
     # This installs everything into a "fake filesystem root", then uninstalls
     # everything and checks that there are no files left over. The purpose of
     # this check is to ensure that: 1) `make install` doesn't fail; 2) `make
@@ -82,7 +82,7 @@ freebsd_task:
     build_script: &build_script
         - make -j${JOBS} --keep-going all test
     test_script: &test_script
-        - RUST_TEST_THREADS=${JOBS} RUST_BACKTRACE=1 make -j${JOBS} ci-check
+        - RUST_TEST_THREADS=${JOBS} RUST_BACKTRACE=1 make -j${JOBS} NEWSBOAT_RUN_IGNORED_TESTS=1 ci-check
     # For explanation of what this script does, please see the FreeBSD job
     # above.
     fake_install_script: &fake_install_script

.github/workflows/coveralls.yaml

@@ -42,7 +42,6 @@ jobs:
           sudo apt-get install --assume-yes locales
           echo 'en_US.UTF-8 UTF-8' | sudo tee --append /etc/locale.gen
           echo 'ru_RU.KOI8-R KOI8-R' | sudo tee --append /etc/locale.gen
-          echo 'nl_NL.ISO-8859-1 ISO-8859-1' | sudo tee --append /etc/locale.gen
           echo 'ru_RU.CP1251 CP1251' | sudo tee --append /etc/locale.gen
           sudo locale-gen
 
@@ -68,7 +67,7 @@ jobs:
         run: cargo install grcov
 
       - name: Run tests
-        run: make --jobs=3 ci-check
+        run: make --jobs=3 NEWSBOAT_RUN_IGNORED_TESTS=1 ci-check
 
         # gcov tool from gcc doesn't understand profiling info that LLVM
         # produces, so we trick grcov into using llvm-cov instead. We can't

CONTRIBUTING.md

@@ -53,7 +53,8 @@ even better than an SSD.
 Newsboat can also be [built in Docker](doc/docker.md).
 
 Please follow our [style guide](doc/internal/code-style.markdown) when
-contributing code.
+contributing code. See also [the Newsboat hacker's
+guide](doc/internal/hackers-guide.asciidoc).
 
 Patches can be submitted by sending a 
 [GitHub Pull Request](https://github.com/newsboat/newsboat/pull/new/master) or 

Makefile

@@ -36,6 +36,10 @@ export CARGO_TARGET_DIR:=$(abspath $(CARGO_TARGET_DIR))
 
 CPPCHECK_JOBS?=5
 
+# By default, do not run "ignored" Rust tests -- they require some additional
+# setup on the test machine.
+NEWSBOAT_RUN_IGNORED_TESTS?=0
+
 # compiler
 CXX?=c++
 # Compiler for building executables meant to be run on the
@@ -414,13 +418,25 @@ profclean:
 
 check: test
 	(cd test && ./test --order=rand --rng-seed=time)
-	+$(CARGO) test $(CARGO_TEST_FLAGS)
+	$(CARGO) test $(CARGO_TEST_FLAGS)
+	if [ $(NEWSBOAT_RUN_IGNORED_TESTS) -eq 1 ] ; then $(CARGO) test $(CARGO_TEST_FLAGS) -- --ignored ; fi
 
 ci-check: test
         # We want to run both C++ and Rust tests, but we also want this entire
         # command to fail if one of the test suites fails. That's why we store
         # the C++'s exit code and chain it to Rust's in the end.
-	$(CARGO) test $(CARGO_TEST_FLAGS) --no-fail-fast ; ret=$$? ; cd test && ./test --order=rand --rng-seed=time && exit $$ret
+	$(CARGO) test $(CARGO_TEST_FLAGS) --no-fail-fast ; \
+	ret1=$$? ; \
+	\
+	ret2=0 ; \
+	if [ $(NEWSBOAT_RUN_IGNORED_TESTS) -eq 1 ] ; \
+		then $(CARGO) test $(CARGO_TEST_FLAGS) --no-fail-fast -- --ignored ; ret2=$$? ; \
+		fi ; \
+	\
+	cd test && \
+	./test --order=rand --rng-seed=time && \
+	\
+	[ \( $$ret1 -eq 0 \) -a \( $$ret2 -eq 0 \) ]
 
 # miscellaneous stuff
 

doc/hackers-guide.asciidoc → doc/internal/hackers-guide.asciidoc

@@ -1,20 +1,25 @@
-= The Newsbeuter Hacker's Guide
+= The Newsboat Hacker's Guide
 Andreas Krennmair <ak@newsbeuter.org>
+Alexander Batishev <eual.jp@gmail.com>
+
+**WARNING**: this document is being slowly updated. See the date of each
+section to get a feel of how up-to-date and reliable is the information in it.
 
 == Introduction
 
-This is the "hacker's guide" to newsbeuter. It describes the overall architecture
-of newsbeuter, the most important design decisions and some other noteworthy things
-that don't belong anywhere else.
+This is the "hacker's guide" to Newsboat. It describes the overall architecture
+of the program, the most important design decisions, and some other noteworthy
+things that don't belong anywhere else.
 
-This guide assumes that you know about Unix programming with C++,
-multithreading, and some more stuff. This is not for end users, so if you don't
-have C++ programming experience, then this is not for you.
+This guide assumes that you know about Unix programming, C++, Rust,
+multithreading, and some more stuff. This is not for end users.
 
 == Architecture
 
 === Classes
 
+**Status of this section:** incomplete (January 2021).
+
 This section describes the different classes and their purpose.
 
 *class cache*: the persistence backend of newsbeuter. It takes rss_item and
@@ -105,13 +110,17 @@ htmlrenderer class.
 
 === Interaction
 
+**Status of this section:** yet unwritten :)
+
 TODO: describe interaction between classes.
 
 
 == Design Decisions
 
 === Use text file as configuration
 
+**Status of this section:** the idea holds (January 2021).
+
 The "classical" text tools, like vim, slrn and mutt, are all configurable
 solely via text files. Newsbeuter follows the same spirit, especially since the
 other prominent RSS feed readers for the text console primarily encourage
@@ -122,6 +131,9 @@ crude menus that are somehow hard to use.
 
 === Keep a good balance of customizability
 
+**Status of this section:** the idea holds, but Newsboat arguably failed this
+goal (January 2021).
+
 The problem with user wishes is that too many people demand a possibility to
 customize this bell or that whistle within newsbeuter. Often, these
 possibilities only have a very limited purpose, and their value is in no
@@ -133,6 +145,9 @@ to read. With too much customizability, this goal would be in danger.
 
 === Why C++ and not C, [insert your favorite language], ...?
 
+**Status of this section:** partially holds, but we're moving to Rust (January
+2021).
+
 C++ has many advantages compared to other programming languages.
 
 - C++ is backwards-compatible to C. That means we can theoretically use all the
@@ -151,6 +166,9 @@ useable language during the development process.
 
 === Use the full potential of modern C++
 
+**Status of this section:** mostly holds, but new stuff should be written in
+Rust if possible (January 2021).
+
 The C++ standard library comes with an extensive set of algorithms and data
 structures. Developers are encouraged to use especially the data structures,
 because the available container classes are standardized, their behaviour and
@@ -165,6 +183,8 @@ and *class stflpp*.
 
 === Getting a detailed debug log
 
+**Status of this section:** up-to-date (January 2021).
+
 If you want to get a detailed debug log from newsbeuter, you only need to run
 newsbeuter with special parameters:
 
@@ -175,17 +195,31 @@ code, so it's only helpful for developers.
 
 === Use (and extend) the unit tests
 
-In the test subdirectory resides a simple unit test to check the most important
-functionality of the newsbeuter internals. These tests build upon the lemon
-test framework which comes bundled as test/lemon.h with newsbeuter. Run "make
-test" to build the tests, the result is a binary called "test" within the test
-subdirectory. Run it and see whether everything still works as expected. Run
-"make clean-test" to clean up after the tests.
+**Status of this section:** up-to-date (January 2021).
+
+C++ tests are in the _test_ subdirectory. They're using
+https://github.com/catchorg/Catch2/tree/devel/docs[the Catch2 framework]. Rust
+tests are split: unit tests are in `mod tests` sub-modules under
+_rust/libnewsboat/src_, while integration tests are under
+_rust/libnewsboat/tests_. Rust's test runner is multi-threaded, so we use
+integration testing when we want to do the checks in a separate process.
+
+In addition to `TMPDIR=/dev/shm make -j5 PROFILE=1 check` incantation that's
+already explained in the contributing guidelines, you should know about the
+`NEWSBOAT_RUN_IGNORED_TESTS` variable. Like `PROFILE`, it can be set to `1`.
+When set, it enables tests that require some additional prerequisites:
+
+- locales:
+    - `en_US.UTF-8`
+    - `ru_RU.CP1251`
+    - `ru_RU.KOI8-R`
 
 == Keys
 
 === Unused keys
 
+**Status of this section:** probably outdated (January 2021).
+
 	      _b_____hi_____________wxyz
 	      _B_____HI__LM_____STUVWXYZ
 	Ctrl- A__D___HIJ____O_Q_S___W_Y_

docker/ubuntu_18.04-build-tools.dockerfile

@@ -82,7 +82,6 @@ RUN addgroup --gid 1000 builder \
 RUN apt-get install locales \
     && echo 'en_US.UTF-8 UTF-8' >> /etc/locale.gen \
     && echo 'ru_RU.KOI8-R KOI8-R' >> /etc/locale.gen \
-    && echo 'nl_NL.ISO-8859-1 ISO-8859-1' >> /etc/locale.gen \
     && echo 'ru_RU.CP1251 CP1251' >> /etc/locale.gen \
     && locale-gen
 ENV LANG en_US.UTF-8

docker/ubuntu_18.04-i686.dockerfile

@@ -69,7 +69,6 @@ RUN apt-get update \
     && apt-get install locales \
     && echo 'en_US.UTF-8 UTF-8' >> /etc/locale.gen \
     && echo 'ru_RU.KOI8-R KOI8-R' >> /etc/locale.gen \
-    && echo 'nl_NL.ISO-8859-1 ISO-8859-1' >> /etc/locale.gen \
     && echo 'ru_RU.CP1251 CP1251' >> /etc/locale.gen \
     && locale-gen
 ENV LANG en_US.UTF-8

docker/ubuntu_20.04-build-tools.dockerfile

@@ -82,7 +82,6 @@ RUN addgroup --gid 1000 builder \
 RUN apt-get install locales \
     && echo 'en_US.UTF-8 UTF-8' >> /etc/locale.gen \
     && echo 'ru_RU.KOI8-R KOI8-R' >> /etc/locale.gen \
-    && echo 'nl_NL.ISO-8859-1 ISO-8859-1' >> /etc/locale.gen \
     && echo 'ru_RU.CP1251 CP1251' >> /etc/locale.gen \
     && locale-gen
 ENV LANG en_US.UTF-8

docker/ubuntu_20.10-build-tools.dockerfile

@@ -82,7 +82,6 @@ RUN addgroup --gid 1000 builder \
 RUN apt-get install locales \
     && echo 'en_US.UTF-8 UTF-8' >> /etc/locale.gen \
     && echo 'ru_RU.KOI8-R KOI8-R' >> /etc/locale.gen \
-    && echo 'nl_NL.ISO-8859-1 ISO-8859-1' >> /etc/locale.gen \
     && echo 'ru_RU.CP1251 CP1251' >> /etc/locale.gen \
     && locale-gen
 ENV LANG en_US.UTF-8

rust/libnewsboat-ffi/src/utils.rs

@@ -82,6 +82,11 @@ mod bridged {
         fn podcast_mime_to_link_type(mime_type: &str, result: &mut i64) -> bool;
 
         fn run_program(argv: &Vec<String>, input: &str) -> String;
+
+        fn translit(tocode: &str, fromcode: &str) -> String;
+        fn convert_text(text: &[u8], tocode: &str, fromcode: &str) -> Vec<u8>;
+        fn utf8_to_locale(text: &str) -> Vec<u8>;
+        fn locale_to_utf8(text: &[u8]) -> String;
     }
 
     extern "C++" {