fixupcontrol

#!/usr/bin/python3
#(C) 2017-2018 Peter Michael Green <plugwash@debian.org>
#This software is provided 'as-is', without any express or implied warranty. In
#no event will the authors be held liable for any damages arising from the use
#of this software.
#
#Permission is granted to anyone to use this software for any purpose, including
#commercial applications, and to alter it and redistribute it freely, subject to
#the following restrictions:
#
#1. The origin of this software must not be misrepresented; you must not claim.
#that you wrote the original software. If you use this software in a product,
#an acknowledgment in the product documentation would be appreciated but is.
#not required.
#
#2. Altered source versions must be plainly marked as such, and must not be
#misrepresented as being the original software.
#
#3. This notice may not be removed or altered from any source distribution.

import sys
import re
import subprocess
import collections
from difflib import SequenceMatcher
from difflib import unified_diff
from afpg_util import mergelists
import deb822

filetofix = sys.argv[1]
tempdir = sys.argv[2]

f = open(filetofix,'rb')
lines = f.readlines()
f.close()

ld = []
lm = []
lu = []

mode = 0; # 0: unconflicted text 1: downstream text 2: mergehead text 3: upstream text

#downstream = set()
#mergehead = set()
#upstream = set()
#deferred = []

#def writesplitline(f,line):
#	hasnewline = False
#	if line[-1:] == b"\n":
#		line = line[:-1]
#		hasnewline = True
#	#print(repr(line[-1:]))
#	linesplit = re.match(rb'^(\s*[^\s]*)(.*)$',line).groups()
#	
#	f.write(b'[1]'+linesplit[0]+b"\n")
#	f.write(b'[2]'+linesplit[1]+b"\n")
#	if hasnewline:
#		f.write(b'[3]'+linesplit[0]+b"\n")
#	else:
#		f.write(b'[3]'+linesplit[0])
#
#for stage in range(0,2):

for line in lines:
		#print(repr(line))
		if (line.startswith(b'<<<<<<< ')):
			if (mode == 0):
				mode = 1
				#print('found <<<<<<< switching to mode 1')
			else:
				print('broken diff3, unexpected <<<<<<<', file=sys.stderr)
				sys.exit(1);
		elif (line.startswith(b'||||||| ')):
			if (mode == 1):
				mode = 2
				#print('found ||||||| switching to mode 2')
			else:
				print('broken diff3, unexpected |||||||', file=sys.stderr)
				sys.exit(1);
		elif (line == b'=======\n'):
			if (mode == 2):
				mode = 3
				#print('found ======= switching to mode 3')
			else:
				print('broken diff3, unexpected =======', file=sys.stderr)
				sys.exit(1);
		elif (line.startswith(b'>>>>>>> ')):
			if (mode == 3):
				#print('found >>>>>>> switching to mode 0')
				mode = 0
			else:
				print('broken diff3, unexpected >>>>>>>', file=sys.stderr)
				sys.exit(1);
		else:
			if ((mode == 0) or (mode == 1)):
				ld.append(line)
			if ((mode == 0) or (mode == 2)):
				lm.append(line)
			if ((mode == 0) or (mode == 3)):
				lu.append(line)


#def trymerge(ld,lm,lu):
#	fd = open(tempdir+'/dtmp','wb')
#	fm = open(tempdir+'/mtmp','wb')
#	fu = open(tempdir+'/utmp','wb')
#	
#	for line in ld:
#		writesplitline(fd,line)
#	
#	for line in lm:
#		writesplitline(fm,line)
#	
#	for line in lu:
#		writesplitline(fu,line)
#	
#	fd.close()
#	fm.close()
#	fu.close()

#	command = ['merge',tempdir+'/dtmp',tempdir+'/mtmp',tempdir+'/utmp']
#	print(command, flush=True)
#	return subprocess.call(command)
#
#def splitonbuilddeps(lines):
#	bbd = []
#	bd = []
#	abd = []
#	mode = 0;
#	for line in lines:
#		if (mode == 0):
#			if line.startswith(b'Build-Depends:'):
#				mode = 1
#				bd.append(line)
#			else:
#				bbd.append(line)
#		elif (mode == 1):
#			if (line.startswith(b' ') or line.startswith(b"\t")) and not line.isspace():
#				bd.append(line)
#			else:
#				mode = 2
#				abd.append(line)
#		else:
#			abd.append(line)
#	if mode != 2:
#		print('failed to split build-depends from rest of control content')
#		sys.exit(1);
#	return (bbd,bd,abd)

def parsedeps(lines):
	text = ''.join(lines)
	deplist = text.split(',')
	depdict = collections.OrderedDict()
	for depraw in deplist:
		dep = depraw.strip()
		p = depraw.find(dep)
		whitespacebefore = depraw[:p]
		whitespaceafter = depraw[p+len(dep):]
		#print(repr(whitespacebefore)+' '+repr(dep)+' '+repr(whitespaceafter))
		#print(dep)
		match = re.match('[a-zA-Z0-9+-.]*',dep)
		#print(repr(match.span()))
		#sys.exit(1)
		splitpos = match.span()[1]
		#print(dep)
		meta = dep[splitpos:]
		dep = dep[:splitpos]
		#print((dep,meta))
		#sys.exit(1)
		#split = re.split(b'[^a-zA-Z0-9+-.]',dep,1)
		#if len(split) == 2:
		#	(dep,meta) = split
		#else:
		#	dep = dep
		#	meta = b''
		if dep in depdict:
			depdict[dep].append((meta,whitespacebefore,whitespaceafter))
		else:
			depdict[dep] = [(meta,whitespacebefore,whitespaceafter)]
	#for line in depdict.items():
	#	print(line)
	#sys.exit(1)
	return depdict

def lineendingfactory():
	return b'\n'

def spacefactory():
	return b' '

def parsecontrol(controllist):
	controldict = collections.OrderedDict()
	#split file into paragraphs.
	foundnoncomment = False
	paragraphnocomments = []
	commentsbefore = []
	linecomments = collections.defaultdict(list)
	lineendwhitespace = collections.defaultdict(lineendingfactory)
	fieldstartwhitespace = collections.defaultdict(spacefactory)
	paragraphssplit = []
	for lineno, line in enumerate(controllist):
		linestripped = line.strip()
		if len(linestripped) == 0:
			#blank line
			if foundnoncomment:
				#paragraph complete
				if line == b'\n':
					whitespaceafter = []
				else:
					whitespaceafter = [line[:-1]]
				paragraphssplit.append([paragraphnocomments,commentsbefore,linecomments,whitespaceafter,lineendwhitespace,fieldstartwhitespace])
				foundnoncomment = False
				paragraphnocomments = []
				commentsbefore = []
				linecomments = collections.defaultdict(list)
				lineendwhitespace = collections.defaultdict(lineendingfactory)
				fieldstartwhitespace = collections.defaultdict(spacefactory)
			else:
				commentsbefore.append(line)
		elif (line[0] == ord(b'#')):
			#comment line
			if foundnoncomment:
				linecomments[lastnoncommentline].append(line)
			else:
				commentsbefore.append(line)
		elif ((0x21 <= line[0] <= 0x39) or (0x3B <= line[0] <= 0x7F)):
			#field line
			linestripped = line.rstrip()
			lews = line[len(linestripped):]
			linesplit = linestripped.split(b':',1)
			if len(linesplit) < 2:
				print('failed to find colon on line '+str(lineno))
				sys.exit()
			valuewithwhitespace = linesplit[1]
			i = 0
			#print(repr(linestripped))
			#print(repr(valuewithwhitespace))
			while (i < len(valuewithwhitespace)) and ((valuewithwhitespace[i] == ord(b' ') or valuewithwhitespace[i] == ord(b"\t"))):
				i += 1
			if (i != 1) or (valuewithwhitespace[0] != ord(b' ')):
				valuewithoutwhitespace = valuewithwhitespace[i:]
				if valuewithoutwhitespace != b'':
					linestripped = linesplit[0]+b': '+valuewithoutwhitespace
				else:
					linestripped = linesplit[0]+b':'
				fieldstartwhitespace[linestripped] = valuewithwhitespace[:i]
				#print(repr(linestripped)+' '+repr(fieldstartwhitespace[linestripped]))
			paragraphnocomments.append(linestripped)
			lineendwhitespace[linestripped] = lews
			lastnoncommentline = linestripped
			foundnoncomment = True
		elif (line[0] == ord(b' ')) or (line[0] == ord(b'\t')):
			#continuation line
			linestripped = line.rstrip()
			paragraphnocomments.append(linestripped)
			lineendwhitespace[linestripped] = line[len(linestripped):]
			lastnoncommentline = linestripped
			foundnoncomment = True
		else:
			print('unexpected first character on line '+str(lineno))
			sys.exit(1)
	if foundnoncomment:
		#paragraph complete
		paragraphssplit.append([paragraphnocomments,commentsbefore,linecomments,[],lineendwhitespace,fieldstartwhitespace])
	else:
		commentsafter = paragraphssplit[-1][3]
		if len(commentsafter) == 0:
			commentsafter = [b'\n']
		else:
			commentsafter[-1] += b'\n'
		paragraphssplit[-1][3] = commentsafter + commentsbefore
	#for paragraph in deb822.Deb822.iter_paragraphs(controllist,shared_storage=False):
	for (paragraphnocomments,commentsbefore,linecomments,commentsafter,lineendwhitespace,fieldstartwhitespace) in paragraphssplit:
		#paragraphregen = str(paragraph).encode('utf-8').splitlines(True)
		titleline = paragraphnocomments[0]
		paragraph = deb822.Deb822(paragraphnocomments)
		#if titleline == b'Package: @DEV_PKG@':
		#	print(repr(paragraph['Depends']))
		#print(titleline)
		if titleline in controldict:
			print('duplicate paragraphs')
			sys.exit(1)
		# second item in tuple is reserved for storing extra information
		# to facilitate round trip conversion
		controldict[titleline] = (paragraph,(commentsbefore,linecomments,commentsafter,lineendwhitespace,fieldstartwhitespace))
		#print(paragraph.__class__)
		#sys.exit(1)
		
	return controldict

def regencontrol(controldict):
	controllist = []
	#regenextra is not currently used, it is there in case I need to store
	#extra stuff in future to allow round-tripping.
	for titleline, (paragraph, (commentsbefore,linecomments,commentsafter,lineendwhitespace,fieldstartwhitespace)) in controldict.items():

		if controllist != []:
			if controllist[-1][-1:] != b'\n':
				controllist[-1] += b'\n'
			else:
				controllist += [b'\n']
		controllist += commentsbefore
		paragraphregen = str(paragraph).encode('utf-8').splitlines(True)
		#print(paragraph.__class__)
		#sys.exit(1)
		for line in paragraphregen:
			linestripped = line.rstrip()
			#if titleline == b'Package: @DEV_PKG@':
			#	print(repr(linestripped)+' '+repr(lineendwhitespace[linestripped]))
			if fieldstartwhitespace[linestripped] == b' ':
				controllist.append(linestripped+lineendwhitespace[linestripped])
			else:
				linesplit = line.split(b':',1)
				controllist.append(linesplit[0]+b':'+fieldstartwhitespace[linestripped]+linesplit[1].strip()+lineendwhitespace[linestripped])
			controllist += linecomments[linestripped]
		controllist += commentsafter
	
	return controllist

#print(repr(lu[0]))
#print(repr(luregen[0]))

def failifmismatch(a,b,fna,fnb):
	if luregen != lu:
		print('cannot round trip control data')
		diff = unified_diff([repr(s)+'\n' for s in lu],[repr(s)+'\n' for s in luregen],fna,fnb)
		for line in diff:
			print(line.rstrip())
		sys.exit(1)

duparsed = parsecontrol(lu)
luregen = regencontrol(duparsed)
failifmismatch(lu,luregen,filetofix+' (upstream)',filetofix+' (upstream regenerated)')

dmparsed = parsecontrol(lm)
lmregen = regencontrol(dmparsed)
failifmismatch(lm,lmregen,filetofix+' (mergehead)',filetofix+' (mergehead regenerated)')

ddparsed = parsecontrol(ld)
ldregen = regencontrol(ddparsed)
failifmismatch(ld,ldregen,filetofix+' (downstream)',filetofix+' (downstream regenerated)')

def merge3sets(sd,sm,su):
	dsadded = sd - sm
	dsremoved = sm - sd
	usadded = su - sm
	usremoved = sm - su
	#print('dsadded: '+repr(dsadded))
	#print('dsremoved: '+repr(dsremoved))
	#print('usadded: '+repr(usadded))
	#print('usremoved: '+repr(usremoved))

	#sys.exit(1)

	finalset = (sm | usadded | dsadded) - usremoved - dsremoved
	#print('finalset: '+repr(finalset))
	#print(sm-dsremoved)
	return finalset

def mergekeysintoset(dd,dm,du):
	#extract sets from dictionaries.
	sd = set(dd.keys())
	sm = set(dm.keys())
	su = set(du.keys())
	#print('downstream: '+repr(sd))
	#print('mergehead: '+repr(sm))
	#print('upstream: '+repr(su))
	return merge3sets(sd,sm,su)

def merge3ordereddicts(dd,dm,du,dicttype,conflicthandler,tag):
	finalset = mergekeysintoset(dd,dm,du)
	#print("finalset: "+repr(finalset))
	final = dicttype()
	for key in mergelists(list(du.keys()),b = list(dd.keys())):
		if key in finalset:
			if key not in dd:
				final[key] = du[key]
			elif key not in du:
				final[key] = dd[key]
			elif du[key] == dd[key]:
				final[key] = du[key]
			elif du[key] == dm[key]:
				final[key] = dd[key]
			elif dd[key] == dm[key]:
				final[key] = du[key]
			else:
				value = conflicthandler(key,dd[key],dm[key],du[key],tag)
				#print('setting '+repr(key)+' to '+repr(value))
				final[key] = value
	return(final)

#this decoder is very simplistic, it handles the most common case, it could
#certainly be improved to handle other cases if that proves nessacery in the future.
depmetapattern = re.compile('(?:([ \t]*)\(([<>= a-zA-Z0-9:.+\-~]*)\))?(?:([ \t]*)\[([ a-z0-9\!\-]*)\])?')
def decodedepmeta(meta):
	m = re.fullmatch(depmetapattern,meta)
	if m is None:
		print('unable to decode dependency metadata '+repr(meta))
		return None
	#print(m.groups())
	(vrws,versionrestriction,alws,archlist) = m.groups()
	if vrws is None:
		vrws = " "
	if alws is None:
		alws = " "
	return (versionrestriction,archlist,vrws,alws)

def parsearchlist(alraw):
	if alraw is None:
		alparsed = []
		alpositive = False
	else:
		alparsed = alraw.split(' ')
		alpositive = (alparsed[0][0] != '!')
		if not alpositive:
			alparsed = [s[1:] for s in alparsed]
	return (alpositive,alparsed)

def makearchset(positive,parsed,knownarchset):
	if positive:
		return set(parsed)
	else: 
		return ( { ... } | knownarchset) - set(parsed)

def mergearchlists(downstream,mergehead,upstream):
	if downstream == mergehead:
		return upstream
	elif upstream == mergehead:
		return downstream
	(dspositive,dsparsed) = parsearchlist(downstream)
	(mhpositive,mhparsed) = parsearchlist(mergehead)
	(uspositive,usparsed) = parsearchlist(upstream)
	knownarchlist = mergelists(mergelists(dsparsed,usparsed),mhparsed)
	knownarchset = set(knownarchlist)
	dsset = makearchset(dspositive,dsparsed,knownarchset)
	mhset = makearchset(mhpositive,mhparsed,knownarchset)
	usset = makearchset(uspositive,usparsed,knownarchset)
	rset = merge3sets(dsset,mhset,usset)
	rpositive = (... not in rset)
	rlist = []
	for arch in knownarchlist:
		if  rpositive:
			if arch in rset:
				rlist.append(arch)
		else:
			if arch not in rset:
				rlist.append("!"+arch)
	if len(rlist) == 0:
		if rpositive:
			#afaict there is no canonical way to represent an empty positive architecture list, use a fake architecture "none"
			return "none"
		else:
			#an empty negative architecture list is simply represented as no architecture list at all
			#which we represent by the python "None" (since that is what our regex gives us)
			return None
	else:
		return " ".join(rlist)

def handledepconflicts(key,downstream,mergehead,upstream,tag):
	(paragraph,field) = tag
	if set(p[0] for p in upstream) == set(p[0] for p in downstream):
		return upstream
	#print(repr(dudep[dep]))
	#print(repr(dudep[dep][0]))
	if (len(upstream) == 1) and (len(downstream) == 1) and (len(mergehead) == 1):
		downstreammeta = downstream[0][0]
		mergeheadmeta = mergehead[0][0]
		(upstreammeta,whitespacebefore,whitespaceafter) = upstream[0]
		downstreammetadecoded = decodedepmeta(downstreammeta)
		mergeheadmetadecoded = decodedepmeta(mergeheadmeta)
		upstreammetadecoded = decodedepmeta(upstreammeta)
		if (downstreammetadecoded is not None) and (mergeheadmetadecoded is not None) and (upstreammetadecoded is not None):
			if downstreammetadecoded[0] == mergeheadmetadecoded[0]:
				versionrestriction = upstreammetadecoded[0]
			elif upstreammetadecoded[0] == mergeheadmetadecoded[0]:
				versionrestriction = downstreammetadecoded[0]
			else:
				print('merging of version restrictions is not currently supported')
				versionrestriction = False
			archlist = mergearchlists(downstreammetadecoded[1],mergeheadmetadecoded[1],upstreammetadecoded[1])
			if (versionrestriction is not False) and (archlist is not False):
				meta = ''
				if versionrestriction is not None:
					meta = upstreammetadecoded[2]+'('+versionrestriction+')'
				if archlist is not None:
					meta += upstreammetadecoded[3]+'['+archlist+']'
				return [(meta,whitespacebefore,whitespaceafter)]
	else:
		print('automatic dependency merging is not currently supported when there are multiple dependencies on the same package')
	print('upstream and downstream versions of paragraph '+repr(paragraph)+' have '+field+' on '+key+' with different metadata and metadata could not be automatically merged, upstream metadata is '+repr(list(p[0] for p in upstream))+' downstream metadata is '+repr(list(p[0] for p in downstream)))
	exit(1)

def handledepfieldconflicts(key,downstream,mergehead,upstream,tag):
	field = key
	paragraph = tag
	#print(paragraph)
	#print("downstream: "+repr(downstream))
	#print("mergehead: "+repr(mergehead))
	#print("upstream: "+repr(upstream))
	dddeps = parsedeps(downstream)
	dmdeps = parsedeps(mergehead)
	dudeps = parsedeps(upstream)
	#print(repr(downstream))
	#print(repr(dddeps))
	#sys.exit(1)

	final = merge3ordereddicts(dddeps,dmdeps,dudeps,collections.OrderedDict,handledepconflicts,(paragraph,field))
	firstdep = True
	regenerateddeps = []
	#print(repr(final))
	queued = ''
	for (dep,meta) in final.items():
		for (metaentry,whitespacebefore,whitespaceafter) in meta:
			line = ''
			if firstdep:
				#print(repr(whitespacebefore))
				firstdep = False
			else:
				regenerateddeps[-1] += queued
			#print(repr(dep)+' '+repr(metaentry))
			line += whitespacebefore + dep + metaentry
			#we don't want to add this stuff for the last line, so queue it here and
			#add it on the next iteration
			queued =  whitespaceafter+ ','
			regenerateddeps.append(line)
	#convert result to a single string
	regenerateddeps = ''.join(regenerateddeps)
	#print(repr(upstream))
	#print()
	#print(repr(regenerateddeps))
	return regenerateddeps


def handlefieldconflicts(key,downstream,mergehead,upstream,tag):
	field = key
	paragraph = tag
	if field in {'Depends','Recommends','Suggests','Conflicts','Build-Depends','Build-Depends-Arch','Build-Depends-Indep','Build-Conflicts','Breaks'}:
		return handledepfieldconflicts(key,downstream,mergehead,upstream,tag)
	if field in {'Architecture'}:
		# handle the "none" psuedo-architecture specially, we use it in raspbian to disable binary packages
		if (downstream == "none") or (upstream == "none"):
			return "none"
		else:
			return mergearchlists(downstream,mergehead,upstream)
	else:
		print('conflict in field '+repr(field)+' of paragraph '+repr(tag))
		print('downstream content '+repr(downstream))
		print('mergehead content '+repr(mergehead))
		print('upstream content '+repr(upstream))
		sys.exit(1)



def handleparagraphconflicts(key,downstream,mergehead,upstream,tag):
	regenextra = upstream[1]
	if downstream[0] == upstream[0]:
		result = upstream[0]
	else:
		result = merge3ordereddicts(downstream[0],mergehead[0],upstream[0],deb822.Deb822,handlefieldconflicts,key)
	return (result,regenextra)

finalparagraphs = merge3ordereddicts(ddparsed,dmparsed,duparsed,collections.OrderedDict,handleparagraphconflicts,None)
finalregen = regencontrol(finalparagraphs)

#for titleline, (paragraph, regenextra) in finalparagraphs.items():
#	print(paragraph.__class__)

f = open(filetofix+'.new','wb')

for line in finalregen:
	f.write(line)

f.close()

command = ['mv',filetofix+'.new',filetofix]
print(command, flush=True)
if (subprocess.call(command) != 0): 
    print('moviing result into place failed')
    sys.exit(1)