old_flifcrush

#!/usr/bin/env python3

#    flifcrush - tries to reduce FLIF files in size
#    Copyright (C) 2016  Matthias Krüger, Jon Sneyers

#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2, or (at your option)
#    any later version.

#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.

#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA


import subprocess
import sys
import os
from PIL import Image
from collections import Counter
from collections import namedtuple
import argparse
from itertools import chain # combine ranges

import random # getrandomfilename
import string # getrandomfilename
__author__ = 'Matthias "matthiaskrgr" Krüger'

parser = argparse.ArgumentParser()
parser.add_argument("inpath", help="file or path (recursively) to be converted to flif", metavar='N', nargs='+', type=str)
parser.add_argument("-o", "--options", help="extra command line options (e.g. -o-N for non-interlaced)", metavar='OPTIONS', nargs=1, type=str)
parser.add_argument("-c", "--compare", help="compare to default flif compression", action='store_true')
args = parser.parse_args()

COMPARE = (args.compare)
INPATHS = args.inpath

if args.options:
    OPTIONS = args.options[0].split()
else:
    OPTIONS = []

# make these global to access them easily inside functions 
global size_before_glob, size_after_glob, files_count_glob, size_flifdefault_glob
size_before_glob = 0 # size of all images we process
size_after_glob = 0 # size of all flifs we generated
files_count_glob = 0  # number of files
size_flifdefault_glob = 0 # size of all images converted with flif default parameters


# colors for stdout
txt_ul = TXT_UL = '\033[04m' # underline
txt_res = TXT_RES = '\033[0m' #reset


#BRUTEFORCE = (args.bruteforce)
global output_best
output_best="none"
global arr_index
global progress_array
arr_index = 0
#progress_array=["|", "/", "-", "\\",]
#progress_array=[".", "o", "0", "O", "O", "o", "."]
progress_array=[" ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃", "▁"]
arrlen=len(progress_array)

# prints activity indicator (some kind of ascii 'animation')
def showActivity(func_arg, size_new):
	global arr_index
	arr_index+=1
	if (arr_index == arrlen):
		arr_index = 0
	diff_best = best_dict['size'] - size_new
	sys.stderr.write(progress_array[arr_index] + " " + str(count) + ": "  + str(func_arg) +  ", size: " + str(size_new) + " b            \r")

# save .flif file that had the best combination of parameters 
def save_file():
	global output_best
	flif2flif = False # default, we need extra parameter if we convert .flif to -clif
	# if the condition is false, we didn't manage to reduce size
	if output_best != "none":
		OUTFILE=".".join(INFILE.split(".")[:-1])+".flif" # split by ".", rm last elm, join by "." and add "flif" extension

		if (OUTFILE == INFILE): # most likely flif fo flif crushing
			flif2flif = True
			OUTFILE=get_rand_filename()

		with open(OUTFILE, "w+b") as f:
			f.write(output_best)
			f.close

		size_flif=os.path.getsize(OUTFILE)
		size_orig=os.path.getsize(INFILE)

		if (flif2flif): # overwrite INFILE with OUTFILE
			os.remove(INFILE)
			os.rename(OUTFILE, INFILE) # rename outfile to infile

		# print some numbers
		global size_after_glob
		size_after_glob += size_flif
		size_diff = size_orig - best_dict['size']

		print("\033[K", end="")
		print("reduced from " + str(size_orig) + " b to "+ str(best_dict['size']) + " ( -"+ str(size_diff) + " b, "+ str((( best_dict['size'] - size_orig)/ size_orig )*100)[:6] + " %) " + str(count) +  " flif calls.\n\n")
		#print("reduced from {size_orig}b to {size_flif}b ({size_diff}b, {perc_change} %) via \n [{bestoptim}] and {cnt} flif calls.\n\n".format(size_orig = os.path.getsize(INFILE), size_flif=size_flif, size_diff=(size_flif - size_orig), perc_change=str(((size_flif-size_orig) / size_orig)*100)[:6],  bestoptim=str("maniac repeats:" + str(best_dict['maniac_repeats']) + " maniac_threshold:" + str(best_dict['maniac_threshold']) + " maniac_min_size:" + str(best_dict['maniac_min_size'])+ " maniac_divisor:" + str(best_dict['maniac_divisor']) + " max_palette_size:" + str(best_dict['max_palette_size']) + " chance-cutoff:" + str(best_dict['chance_cutoff'])  + " chance-alpha:" + str(best_dict['chance_alpha']) +  " ACB:" + str(best_dict['ACB']) + " INTERLACE:" + str(best_dict['INT']) + " PLC:" + str(best_dict['PLC']) + " RGB:" +  str(best_dict['RGB']) +  " A:" + str(best_dict['A'])), cnt=str(count)), end="\r")

		if (best_dict['size'] > size_orig):
			print("WARNING: failed to reduce size")

	else:
		print("\033[K", end="")
		print("WARNING: could not reduce size!")

# generates a name for a file that does not exist in current directory, used for tmp files
def get_rand_filename(): 
	# this prevents accidentally overwriting a preexisting file
	filename =''.join(random.choice(string.ascii_uppercase) for i in range(9))
	while (os.path.isfile(filename)): # if the name already exists, try again
		filename =''.join(random.choice(string.ascii_uppercase) for i in range(9))
	return filename

def pct_of_best(size_new):
	# if best size was 100 and new file is 50, return  50 %
	pct = str(((size_new - best_dict['size']) / best_dict['size'])*100)
	pct = "-0.000" if ("e" in pct) else pct[:6] # due to too-early [:6], '8.509566454608271e-07' would become "8.509"
	return pct

def range_around_default(default, min, max, stepsize):
        result = []
        i = default
        plus = default+stepsize
        minus = default-stepsize
        c = 0
        while plus<=max or minus>=min:
            if plus<=max:
                result += [plus]
                plus += stepsize
            if minus>=min:
                result += [minus]
                minus -= stepsize
            c += 1
            if c>20: # list is getting long, increase stepsize to reach boundaries faster
                stepsize += 1
                c = 0
        return result

def range_above_default(default, min, max, stepsize):
        result = []
        plus = default+stepsize
        c = 0
        while plus<=max:
            result += [plus]
            plus += stepsize
            c += 1
            if c>10: # list is getting long, increase stepsize to reach boundaries faster
                stepsize += 1
                c = 0
        return result

def range_below_default(default, min, max, stepsize):
        result = []
        minus = default-stepsize
        c = 0
        while minus>=min:
            result += [minus]
            minus -= stepsize
            c += 1
            if c>10: # list is getting long, increase stepsize to reach boundaries faster
                stepsize += 1
                c = 0
        return result

def strfield(highlight,parameters,field,shortname):
    if (field == highlight):
        return TXT_UL + shortname + str(parameters[field]) + TXT_RES
    else:
        return shortname + str(parameters[field])

def strfieldbool(highlight,parameters,field,shortname):
    if parameters[field]:
        if (field == highlight):
            return TXT_UL + ' ' + shortname + ' ' + TXT_RES
        else:
            return ' ' + shortname + ' '
    else:
        if (field == highlight):
            return TXT_UL + '(' + shortname + ')' + TXT_RES
        else:
            return '(' + shortname + ')'

def encode_attempt(field,value):
	# globals we modify
	global best_dict
	global count
	global arr_index
	global output_best

	count += 1
	parameters=best_dict.copy()
	parameters[field] = value
	raw_command = [
		flif_binary,
		flif_to_flif,
		('--maniac-repeats=' + str(parameters['maniac_repeats'])),
		('--maniac-threshold=' + str(parameters['maniac_threshold'])),
		('--maniac-divisor=' + str(parameters['maniac_divisor'])),
		('--maniac-min-size=' + str(parameters['maniac_min_size'])),

		('--chance-cutoff=' + str(parameters['chance_cutoff'])),
		('--chance-alpha=' + str(parameters['chance_alpha'])),
		('--max-palette-size=' + str(parameters['max_palette_size']))
		]
	if parameters['no_channel_compact']:
		raw_command += ['--no-channel-compact']
	if str(parameters['force_color_buckets']) != '(-AB)':
		raw_command += [str(parameters['force_color_buckets'])]
	if parameters['no_ycocg']:
		raw_command += ['--no-ycocg']
	if parameters['keep_invisible_rgb']:
		raw_command += ['--keep-invisible-rgb']
	raw_command += [
		INFILE,
		'/dev/stdout'
	] # = raw_command
	raw_command += OPTIONS

	sanitized_command = [x for x in raw_command if x ] # remove empty elements, if any
	#print('running: ', sanitized_command)
	output = subprocess.Popen(sanitized_command, stdout=subprocess.PIPE).stdout.read()
	#size_new = sys.getsizeof(output)
	size_new = len(output)
	showActivity(field + ": " + str(value), size_new)
	if ((best_dict['size'] > size_new) or (count==1)): # new file is smaller // count==1: make sure best_dict is filled with first values we obtain. this way we still continue crushing even if initial N-run does not reduce size smaller than size_orig
		output_best = output
		if (size_orig > size_new):
			size_change = best_dict['size']-size_new
			perc_change = pct_of_best(size_new)
			print("\033[K", end="")
			print(
				 str(count).ljust(4) + " | Options: " +

#				 " maniac:[" +
				 strfield(field,parameters,'maniac_repeats','-R') + ' ' +
				 strfield(field,parameters,'maniac_threshold','-T') + ' ' +
				 strfield(field,parameters,'maniac_min_size','-M') + ' ' +
				 strfield(field,parameters,'maniac_divisor','-D') +
				 ' ' +
#				 "] " + # ] maniac

#				 " chance:[" +
				 strfield(field,parameters,'chance_cutoff','-X') + ' ' +
				 strfield(field,parameters,'chance_alpha','-Z') +
#				 "] " + # ] chance
				 ' ' +

				 strfield(field,parameters,'max_palette_size','-P') + ' ' +
				 strfield(field,parameters,'force_color_buckets','') + ' ' +
				 strfieldbool(field,parameters,'no_channel_compact','-C') + ' ' +
				 strfieldbool(field,parameters,'keep_invisible_rgb','-K') + ' ' +
				 strfieldbool(field,parameters,'no_ycocg','-Y') + ' ' +
				 ' '.join(OPTIONS) + ' '

				 "\t| size " + str(size_new) + "b " +
				 "-" + str(size_change) + "b " +
				 perc_change + " %", flush=True)

		best_dict['size'] = size_new
		best_dict['count'] = count
		best_dict[field] = value
		arr_index = 0



def crush_maniac_repeats(): # -R
	max_attempts = 3
	failed_attempts = 0
	for maniac_repeats in range_around_default(best_dict['maniac_repeats'],0,20,1):
		if encode_attempt('maniac_repeats',maniac_repeats):
			failed_attempts = 0 # reset break-counter
		else: # file is not smaller
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break # break out of loop, we have wasted enough time here


def crush_maniac_threshold(): # -T
	max_attempts = 20
	failed_attempts = 0
	improved = False
	for maniac_threshold in range_above_default(best_dict['maniac_threshold'],5,800,1):
		if encode_attempt('maniac_threshold',maniac_threshold):
			failed_attempts = 0 # reset break-counter
			improved = True
		else: # file is not smaller
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break # break out of loop, we have wasted enough time here
	if improved: return # found something better, no need to check range_below_default
	failed_attempts = 0
	for maniac_threshold in range_below_default(best_dict['maniac_threshold'],5,800,1):
		if encode_attempt('maniac_threshold',maniac_threshold):
			failed_attempts = 0 # reset break-counter
		else: # file is not smaller
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break # break out of loop, we have wasted enough time here


def crush_maniac_divisor(): # -D
	max_attempts = 8
	failed_attempts = 0
	improved=False
	for maniac_divisor in range_above_default(best_dict['maniac_divisor'],1,100000,1):
		if encode_attempt('maniac_divisor',maniac_divisor):
			failed_attempts = 0
			improved=True
		else:
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break
	if improved: return
	failed_attempts = 0
	for maniac_divisor in range_below_default(best_dict['maniac_divisor'],1,100000,1):
		if encode_attempt('maniac_divisor',maniac_divisor):
			failed_attempts = 0
		else:
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break


def crush_maniac_min_size(): # -M
	max_attempts = 5
	failed_attempts = 0
	improved = False
	for maniac_min_size in range_above_default(best_dict['maniac_min_size'],0,30000,5):
		if encode_attempt('maniac_min_size',maniac_min_size):
			failed_attempts = 0
			improved = True
		else:
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break
	if improved: return
	failed_attempts = 0
	for maniac_min_size in range_below_default(best_dict['maniac_min_size'],0,30000,5):
		if encode_attempt('maniac_min_size',maniac_min_size):
			failed_attempts = 0
		else:
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break

def crush_chance_cutoff():
	failed_attempts = 0
	max_attempts=5
	# should be possible to find the best cutoff quickly:
	# if optimal cutoff is higher than the default, compression should be increasingly better as the cutoff grows,
	# so no point to keep attempting higher and higher cutoffs

	for chance_cutoff in range_around_default(best_dict['chance_cutoff'],1,128,1):
		if encode_attempt('chance_cutoff',chance_cutoff):
			failed_attempts = 0
		else:
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break


def crush_chance_alpha(): # -Z
	failed_attempts = 0
	max_attempts=4
	improved=False
	for chance_alpha in range_above_default(best_dict['chance_alpha'],2,128,1):
		if encode_attempt('chance_alpha',chance_alpha):
			failed_attempts = 0
			improved = True
		else:
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break
	if improved: return
	failed_attempts = 0
	for chance_alpha in range_below_default(best_dict['chance_alpha'],2,128,1):
		if encode_attempt('chance_alpha',chance_alpha):
			failed_attempts = 0
		else:
			failed_attempts += 1
			if (failed_attempts >= max_attempts):
				break

def crush_max_palette_size():
	for max_palette_size in [0, inf['colors']-1, inf['colors'], -inf['colors'], 1-inf['colors']]:
		if max_palette_size < 32000 and max_palette_size > -32000:
			encode_attempt('max_palette_size',max_palette_size)

def crush_keep_invisible_rgb():
	for keep_invisible_rgb in True, False:
		encode_attempt('keep_invisible_rgb',keep_invisible_rgb)

def crush_force_color_buckets():
	for force_color_buckets in '(-AB)','-A','-B':
		encode_attempt('force_color_buckets',force_color_buckets)

def crush_no_ycocg():
	for no_ycocg in True, False:
		encode_attempt('no_ycocg',no_ycocg)

def crush_no_channel_compact():
	for no_channel_compact in True, False:
		encode_attempt('no_channel_compact',no_channel_compact)



# make sure we know where flif binary is
flif_binary = ""
try: # search for "FLIF" enviromental variable first
	flif_path = os.environ['FLIF']
	if os.path.isfile(flif_path): # the variable actually points to a file
		flif_binary = flif_path
except KeyError: # env var not set, check if /usr/bin/flif exists
	if (flif_binary == ""):
		if (os.path.isfile("/usr/bin/flif")):
			flif_binary = "/usr/bin/flif"
		elif (os.path.isfile("/usr/share/bin/flif")):
			flif_binary = "/usr/share/bin/flif"
		else:
			print("Error: no flif binary found, please use 'export FLIF=/path/to/flif'")
			os.exit(1)


SUPPORTED_FILE_EXTENSIONS=['png', 'flif'] # @TODO add some more
input_files = []
try: # catch KeyboardInterrupt

	for path in INPATHS: # iterate over arguments
		if (os.path.isfile(path)): # inpath is not a directory but a file
			input_files.append(path) # add to list
		else:  # else walk recursively
			for root, directories, filenames in os.walk(path):
				for filename in filenames:
					if (filename.split('.')[-1] in SUPPORTED_FILE_EXTENSIONS): # check for valid filetypes
						input_files.append(os.path.join(root,filename)) # add to list

# generation of input_files list is done:

	current_file = 0
	for INFILE in input_files: # iterate over every file that we go
		current_file += 1
		file_count_str = "(" + str(current_file) + "/" + str(len(input_files)) + ") " # X/Yth file
		flif_to_flif = ""
		files_count_glob += 1
		#output some metrics about the png that we are about to convert
		inf={'path': INFILE, 'sizeByte': 0, 'colors': 0, 'sizeX': 0, 'sizeY':0, 'px':0, 'filetype': INFILE.split('.')[-1]}

		if (inf['filetype'] == "flif"): # PIL does not know flif (yet...?), so we convert the .flif to .png, catch it, and get the amount of pixels
			flif_to_flif = "-t" # flif needs -t flag in case of flif to flif
			FIFO=get_rand_filename() + ".png" # make sure path does not exist before
			os.mkfifo(FIFO) # create named pipe
			subprocess.Popen([flif_binary, INFILE, FIFO])  # convert flif to png to get pixel data
			im = Image.open(FIFO) # <- png data
			os.remove(FIFO) # remove named pipe
		else:
			im = Image.open(INFILE)

		# @TODO: can we speed this up?
		# just for fun:
		img=[] # will contain px data
		for px in (im.getdata()): # iterate over the pixels of the input image so we can count the number of different colors
			img.append(px)

		inf={'path': INFILE, 'sizeByte': os.path.getsize(INFILE), 'colors': len(Counter(img).items()), 'sizeX': im.size[0], 'sizeY': im.size[1], 'px': im.size[0]*im.size[1], 'filetype': INFILE.split('.')[-1]}

		print(file_count_str + inf['path'] + "; dimensions: "  + str(inf['sizeX']) +"×"+ str(inf['sizeY']) + ", " + str(inf['sizeX']*inf['sizeY']) + " px, " + str(inf['colors']) + " unique colors," + " " + str(inf['sizeByte']) + " b")
		size_orig = inf['sizeByte']
		size_before_glob  += size_orig


		global best_dict
		# these have to be the flif default values
		best_dict={'count': -1,
				'maniac_repeats': 1, # 3
				'maniac_threshold': 64,
				'maniac_min_size': 50,
				'maniac_divisor': 30,
				'max_palette_size': 1024,
				'chance_cutoff': 2,
				'chance_alpha': 19,
				'no_channel_compact': False,
				'force_color_buckets': '(-AB)',  # use the default encoder choice (which is neither -A nor -B)
				'no_ycocg': False,
				'keep_invisible_rgb': False,
				'size': size_orig,
				}


		global count
		count = 0 # how many recompression attempts did we take?
		best_count = 0 # what was the smallest compression so far?

		size_new = size_best = os.path.getsize(INFILE)

		if (COMPARE):  #do a default flif run with no special arguments except those provided
			proc = subprocess.Popen([flif_binary, INFILE, '/dev/stdout'] + OPTIONS, stdout=subprocess.PIPE)
			output_flifdefault = proc.stdout.read()
			#size_flifdefault = sys.getsizeof(output_flifdefault)
			size_flifdefault = len(output_flifdefault)
			size_flifdefault_glob += size_flifdefault

		max_iterations = 20
		it = 0

		# find the best color representation (should be mostly orthogonal to the other encode options)
		crush_max_palette_size()
		crush_force_color_buckets()
		crush_no_ycocg()
		crush_no_channel_compact()
		crush_keep_invisible_rgb()

		while (it < max_iterations):
			sze_beginning = best_dict['size']
			# find the best chance cutoff
			crush_chance_cutoff()
			crush_chance_alpha()
			# try to find a good tree
			crush_maniac_threshold()
			crush_maniac_divisor()
			crush_maniac_min_size()
			crush_maniac_repeats()

			# if iteration didn't reduce anything, stop'
			it+=1
			if (sze_beginning == best_dict['size']):
				break


		if (COMPARE): # how does flifcrush compare to default flif conversion?
			diff_to_flif_byte = best_dict['size'] - size_flifdefault
			if (best_dict['size'] > size_flifdefault):
				print("WARNING: flifcrush failed reducing reducing size better than default flif, please report!")
			diff_to_flif_perc = (((size_flifdefault-best_dict['size']) / best_dict['size'])*100)
			print("\033[K", end="") # clear previous line
			print("\nComparing flifcrush (" + str(best_dict['size']) +" b) to default flif (" + str(size_flifdefault)  + " b): " + str(diff_to_flif_byte) + " b which are " + str(diff_to_flif_perc)[:6] + " %")


		# write final best file
		save_file()

	if (files_count_glob > 1):
		if (COMPARE):
			print("In total, reduced " + str(size_before_glob) + " b to " + str(size_after_glob) + " b, " + str(files_count_glob) + " files , " + str(((size_after_glob - size_before_glob)/size_before_glob)*100)[:6] + "%")
			print("Flif default would have been: " + str(size_flifdefault_glob) + " b")
		else:
			print("In total, reduced " + str(size_before_glob) + " b to " + str(size_after_glob) + " b, " + str(files_count_glob) + " files , " + str(((size_after_glob - size_before_glob)/size_before_glob)*100)[:6] + "%")
except KeyboardInterrupt:
	print("\033[K", end="") # clear previous line
	print("\rTermination requested, saving best file so far...\n")
	try: # double ctrl+c shall quit immediately
		save_file()
		if (files_count_glob > 1):
			if (COMPARE):
				print("In total, reduced " + str(size_before_glob) + " b to " + str(size_after_glob) + " b, " + str(files_count_glob) + " files , " + str(((size_after_glob - size_before_glob)/size_before_glob)*100)[:6] + "%")
				print("Flif default would have been: " + str(size_flifdefault_glob) + " b")
			else:
				print("In total, reduced " + str(size_before_glob) + " b to " + str(size_after_glob) + " b, " + str(files_count_glob) + " files , " + str(((size_after_glob - size_before_glob)/size_before_glob)*100)[:6] + "%")
	except KeyboardInterrupt: # double ctrl+c
		print("\033[K", end="") # clear previous line
		print("Terminated by user.")