lbry-sync-ytdl

#!/bin/sh

: <<=cut

=pod

=head1 NAME

lbry-sync-ytdl - copy videos from centralized media-hosting platforms to LBRY


=head1 SYNOPSIS

B<lbry-sync-ytdl> [-b I<bid>] [-c I<channel>] I<videourl>


=head1 DESCRIPTION

B<lbry-sync-ytdl> copies videos or audio (including playlists) from media hosting sites (such as YouTube) to the LBRY network.

=head1 OPTIONS

=over 4

=item B<-a>, B<--api> I<server>

Custom URL for lbyrnet API (e.g. "https://localhost:5279")


=item B<-b>, B<--bid> I<bid>

Amount of LBC to deposit for each piece of content.


=item B<-c>, B<--channel> I<channel>

Publish to channel I<channel>.


=item B<--dry-run>

Don't publish anything; print out URL if a file with the same name does not exist on LBRY.


=item B<-h>, B<--help>

Show help.


=item B<-l>, B<--license> I<lic>

Use I<lic> as license (overrides information gathered from yt-dlp).


=item B<-L>, B<--language> I<lang>

Use I<lang> as language, such as 'en' or 'uk-UA' (overrides information gathered by yt-dlp).


=item B<--no-optimize-video>

Disable lbrynet's automatic video post-processing with ffmpeg.


=item B<-o>, B<--overwrite>

Upload content even if a LBRY claim with this name already exists.


=item B<--orig-thumbnail>

Use the thumbnail URL provided by the original site, rather than re-uploading to LBRY.


=item B<-q>, B<--quiet>

No info output.


=item B<-Q>, B<--really-quiet>

No info or error output.


=item B<--postprocessor-args> I<args>

Pass arguments I<args> to the post-processor (for example \"-strict -2\" for ffmpeg).


=item B<-v>, B<--version>

Show script version and exit.


=item B<-y>, B<--ytdl-options> I<opts>

Pass arguments I<opts> as options for B<yt-dlp> (e.g. \"--playlist-start=10\").


=back


=head1 EXAMPLE

=over 4

=item lbry-sync-ytdl "https://www.youtube.com/watch?v=dPZTh2NKTm4"

Run the script in test mode and just show the information that youtube-dl managed to extract. 

=item lbry-sync-ytdl "https://www.youtube.com/watch?v=dPZTh2NKTm4" --bid=1.0

Actually publish content and bid 1 LBC. It's recommended to run this only after running the script in test mode. 

This will publish the content anonymously.

=item lbry-sync-ytdl --channel="@MyChannel" "https://www.youtube.com/watch?v=dPZTh2NKTm4" --bid=1.0

Publish content to @MyChannel. 

The channel must already exist on the LBRY network and be owned by the current lbrynet user.

=back


=head1 AUTHORS

Copyright (C) 2020-2023 gardenappl, with contributions from Ole Tange and matthiss.


=head1 LICENSE

Copyright (C) 2012 Free Software Foundation, Inc.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
at your option any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.


=head1 DEPENDENCIES

B<lbry-sync-ytdl> uses B<yt-dlp> (or B<youtube-dl>), B<curl>, B<lbrynet>, and (optionally) B<ffmpeg>.


=head1 SEE ALSO

B<yt-dlp>(1), B<youtube-dl>(1)


=cut



_lbry_sync_ytdl_version="v1.8.3"

set -o errexit


options=y:b:c:hva:pnl:L:oqQ
longopts=ytdl-options:,bid:,channel:,validate-video,help,optimize-video,version,api:,publish,orig-thumbnail,postprocessor-args:,no-optimize-video,license:,language:,overwrite,quiet,really-quiet,dry-run
parsed=$(getopt --options=$options --longoptions=$longopts --name "$0" -- "$@")
eval set -- "$parsed"

while true; do
	case "$1" in
		-a|--api )
			api_server="$2"
			shift 2
			;;
		-h|--help )
			help=y
			shift
			;;
		--orig-thumbnail )
			original_thumbnail=y
			shift
			;;
		-v|--version )
			version=y
			shift
			;;
		--no-optimize-video )
			no_optimize=y
			shift
			;;
		--optimize-video )
			print_info "Warning: --optimize-video option is deprecated, this script automatically optimizes video files. Use --no-optimize-video to cancel."
			shift
			;;
		--validate-video )
			print_info "Warning: --validate-video option is deprecated, this script automatically optimizes video files. Use --no-optimize-video to cancel."
			shift
			;;
		-c|--channel )
			channel="$2"
			shift 2
			;;
		-y|--ytdl-options )
			ytdl_options="$2"
			shift 2
			;;
		--postprocessor-args )
			postprocess_args="$2"
			shift 2
			;;
		-b|--bid )
			bid="$2"
			shift 2
			;;
		-l|--license )
			license="$2"
			shift 2
			;;
		-L|--language )
			language="$2"
			shift 2
			;;
		-o|--overwrite )
			overwrite=y
			shift
			;;
		-q|--quiet )
			quiet_info=y
			shift
			;;
		-Q|--really-quiet )
			quiet_info=y
			quiet_error=y
			shift
			;;
		--dry-run )
			dry_run=y
			shift
			;;
		-- )
			shift
			break
			;;
		* )
			print_error "Programming error: $1"
			exit 1
			;;
	esac
done


print_info() {
	if [ -z "$quiet_info" ]; then
		>&2 echo $@
	fi
}

print_error() {
	if [ -z "$quiet_error" ]; then
		>&2 echo $@
	fi
}


print_help() {
	echo \
"Usage: lbry-sync-ytdl URL [options...]

	-a, --api URL                   Custom URL for lbyrnet API (e.g. \"https://localhost:5279\")
	-b, --bid LBC                   Amount of LBC to deposit for each piece of content
	-c, --channel CHANNEL           Name of publisher channel (e.g. \"@MyCoolChannel\")
	    --dry-run                   Don't publish anything; print out URL if a file with the same name does not exist on LBRY.
	-h, --help                      Print help
	-l, --license LICENSE           Add license information (overrides info from youtube-dl)
	-L, --language LANG             Add language tag, like \"en\" or \"uk-UA\"
	    --no-optimize-video         Upload video as-is, don't let LBRY transcode it
	    --orig-thumbnail            Use original thumbnail URL instead of re-uploading to LBRY/spee.ch
	-o, --overwrite                 Upload content even if a file with the same name exists on LBRY
	-q, --quiet                     No info output
	-Q, --really-quiet              No info or error output
	    --postprocessor-args OPTS   Set custom options for youtube-dl postprocessor, usually FFmpeg (e.g. \"-strict -2\")
	-v, --version                   Print version and exit
	-y, --ytdl-options OPTS         Set custom options for youtube-dl (e.g. \"--playlist-start=10\")

If --bid is not provided, script will NOT publish anything, treat that as a test run.

If --channel is not provided, script will publish content anonymously."
}

if [ -n "$help" ]; then
	print_help
	exit 0
elif [ -n "$version" ]; then
	echo "lbry-sync-ytdl $_lbry_sync_ytdl_version"
	exit 0
elif [ -z "$1" ]; then
	print_help
	exit 1
fi


lbrynet_check_error() {
	case "$1" in
		\{* )
			case "$1" in
				*'"error": '* )
					if message="$(printf '%s' "$1" | jq '.error.message' --exit-status --raw-output)"; then
						print_error "LBRY daemon reported an error."
						print_error "$message"
						return 1
					else
						print_error "Unknown error in LBRY daemon."
						print_error "$1"
						return 1
					fi
			esac
			;;
		* )
			print_error "Unknown error in LBRY daemon."
			print_error "$1"
			return 1
			;;
	esac
}

is_running() {
	case "$1" in
		\{* )
			#If starts with {, assume valid JSON
			case "$1" in
				*'"is_running": true'* )
					return 0
					;;
			esac
			;;
	esac
	return 1
}

lbrynet_start_if_needed() {
	if ! pidof "lbrynet" > /dev/null; then
		print_info "Starting LBRY daemon... "
		lbrynet start 2>/dev/null > /dev/null &
		while sleep 0.5; do
			lbrynet_status=$(lbrynet status)
			if is_running "$lbrynet_status"; then
				print_info "Done."
				break
			fi
		done
	fi
}

get_local_lbrynet_config() {
	lbrynet_config=$(lbrynet settings get)
	lbrynet_check_error "$lbrynet_config" || exit 1
	api_server=$(printf '%s' "$lbrynet_config" | jq -r '.api')
}


if [ -z "$api_server" ]; then
	if lbrynet version > /dev/null 2>&1; then
		lbrynet_start_if_needed
		get_local_lbrynet_config
	else
		print_error "Error: no access to 'lbrynet' executable, and no API URL provided" >&2
		exit 1
	fi
fi

if [ -z "$bid" ] && [ -z "$dry_run" ]; then
	publishing_test=y
fi

if [ -n "$publishing_test" ]; then
	print_info "Note: operating in test mode. Content is not actually being published."
	print_info "Use the --bid option when you're sure you're ready to upload."
fi


info_dir="$(mktemp --directory -t lbry-sync-ytdl-info-XXXXXX)"
download_info_file="$(mktemp -t lbry-sync-ytdl-download-info-XXXXXX.json)"
mkfifo "$info_dir/fifo"
trap 'rm -r "$info_dir" && rm "$download_info_file"' EXIT



# Try to find already existing LBRY upload with this LBRY URL
check_exists() {
	result="$(curl --silent -d"{\"method\": \"resolve\", \"params\": {\"urls\": [\"$1\"]}}" "$api_server")"

	error_name="$(printf '%s' "$result" | jq ".result[\"$1\"].error.name" --raw-output)"
	if [ "$error_name" = 'NOT_FOUND' ]; then
		return 1
	fi
}



if type yt-dlp > /dev/null; then
	ytdl="yt-dlp --compat-options youtube-dl"
else
	ytdl="youtube-dl"
fi


$ytdl "$1" --dump-json > "$info_dir/fifo" &


# Read $info_dir/fifo line by line: every line is a video file

publish() {
	name="$(printf '%s' "$line" | jq '.title' --raw-output)"


	# get name in a format that can be used for LBRY URLs
	publish_name="$(printf '%s' "$name" | tr '[:upper:] ' '[:lower:]-' | tr --delete --complement '0-9a-zA-Z[=-=]' | tr --squeeze-repeats '-')"
	url="$(printf '%s' "$line" | jq .webpage_url --raw-output)"

	if [ -n "$channel" ]; then
		lbry_url="$channel/$publish_name"
	else
		lbry_url="$publish_name"
	fi


	if [ -n "$dry_run" ]; then
		check_exists "$lbry_url" || echo "$url"
		return
	fi

	print_info

	if [ -z "$overwrite" ]; then 
		print_info "Checking if $lbry_url is already published..."
		if check_exists "$lbry_url"; then
			print_info "Skipping $name"
			print_info "Already published: $publish_name"
			skipped=y
			return
		fi
	fi


	print_info "Downloading $name... $url"

	if ! $ytdl "$url" --postprocessor-args "$postprocess_args" --merge-output-format=mp4 $ytdl_options --print-json > "$download_info_file"; then
		ytdl_error=y
		return 1
	fi

	if ! thumbnail_url="$(jq '.thumbnail' --raw-output --exit-status < "$download_info_file")"; then
		print_info "\033[31mWarning: no thumbnail!\033[0m"
	elif [ -z "$bid" ]; then
		print_info "This is a test run, not uploading thumbnail..."
	elif [ -z "$original_thumbnail" ]; then
		print_info "Downloading thumbnail: $thumbnail_url"

		thumb_file="$(mktemp -t youtube-dl-thumb-XXXXXX)"
		curl "$thumbnail_url" --output "$thumb_file" --silent

		print_info "Uploading thumbnail to spee.ch..."
		speech_data="$(curl "https://spee.ch/api/claim/publish" --request POST -F "file=@$thumb_file" --silent)"
		rm "$thumb_file"

		thumbnail_url="$(printf '%s' "$speech_data" | jq '.data.serveUrl' --raw-output)"
		print_info "Uploaded as $thumbnail_url"
	fi

	if [ "$thumbnail_url" != 'null' ]; then
		thumbnail_url="\"$thumbnail_url\""
	fi

	if [ -n "$license" ]; then
		upload_license="\"$license\""
	else
		upload_license='null'
	fi

	if [ -n "$language" ]; then
		upload_language="\"$language\""
	else
		upload_language='null'
	fi

	if [ -n "$channel" ]; then
		upload_channel="\"$channel\""
	else
		upload_channel='null'
	fi

	data="$(jq \
		"{
			name: \"$publish_name\",
			channel_name: $upload_channel,
			tags: .tags,
			languages: ((.formats | map(.language) + [ $upload_language ]) | map(select(. != null)) | unique),
			license: ($upload_license // .license),
			release_time: (.release_timestamp // .timestamp //
				try ((.release_date // .upload_date) | strptime(\"%Y%m%d\") | mktime) catch null
			),
			author: .uploader,
			title: .title,
			thumbnail_url: $thumbnail_url,
			file_path: \"$PWD/\\(._filename)\",
			bid: \"$bid\",
			description: .description,
			optimize_file: ((.vcodec != \"none\") and (\"$no_optimize\" != \"y\")),
			blocking: true
		} | map_values(select(. != null))" < "$download_info_file")"
	# strip null values because lbrynet can't handle them

	if [ -z "$bid" ]; then
		print_info "This content would've been published if you'd supplied the --bid amount:"
		printf '%s' "$data" | jq
	else
		print_info "Publishing \"$name\"..."
		result="$(curl --silent -d"{\"method\": \"publish\", \"params\": $data}" "$api_server")"

		#Print result
		lbrynet_check_error "$result" || return 1

		if url="$(printf '%s' "$result" | jq '.result.outputs | .[] | select(.type == "claim") | .permanent_url' --exit-status --raw-output)"; then
			print_info "Published as $url"
		fi
	fi
}

while read -r line; do
	for i in 1 2 3; do
		if publish "$line"; then
			break;
		elif [ "$i" -lt 3 ]; then
			print_info "Failed to publish, wait a second..."
			sleep 1
			print_info "Attempt number $((i + 1))..."
		else
			print_info "Failed to publish file."
			if [ -n "$ytdl_error" ]; then
				print_info "If you're uploading a playlist, this might mean that some videos were unavailable."
			fi
			exit 1
		fi
	done
done < "$info_dir/fifo"



if [ -n "$bid" ]; then
	print_info "Done. Please wait for a few minutes for claims to be processed by the network."
elif [ -n "$publishing_test" ]; then
	print_info "Done. Please verify that all the data is correct,"
	print_info "then run the command again with the --bid option."
fi

if [ -n "$skipped" ]; then
       print_info "\n\033[31mWarning: some videos got skipped!\033[0m"
       print_info "Some videos were skipped automatically because a video with the exact same upload name already exists on LBRY."
       print_info "Use --overwrite option to upload them."
fi

# Local Variables:
# mode: Shell
# tab-width: 4
# sh-basic-offset: 4
# End: