#!/usr/bin/env python3
import sys

import re
import time
import urllib.request
import os
import pickle
from collections import defaultdict as table

# ~/Загрузки/contest_113_20181218003430.tgz
# argv: ~/Загрузки/contest_113_20181218003430.tgz

# Run selection
#	Download all runs
#	Download selected runs (20)
# X	Download OK runs
#	Download OK and PR runs
#	Download OK, PR, RJ, IG, PD, DQ runs
#File name pattern
#	Use Contest Id
# X	Use run number
#	Use user Id
# X	Use user Login
#	Use user Name
# X	Use problem short name
#	Use programming language short name
# X	Use submit time
# X	Use source language or content type suffix

# No directory structure

def mkt(st):
    return time.mktime(time.strptime(st,"%Y%m%d%H%M%S"))

def table_list(): return table(list)
def table_dict(): return table(dict)
def table_set(): return table(set)

CF = sys.argv[1]
for sfx in sys.argv[2:]:
    CFDELETE = "{}.{}.tmp".format(CF,sfx)
    if os.path.isfile(CFDELETE):
        os.unlink(CFDELETE)

URL = "http://uneex.ru/LecturesCMC/PythonIntro2019" #?action=raw
RAW = "?action=raw"
reBase=re.compile(b".*\[\[(/\w+).*<<Date.*<<Date.(.*)T.*")
reTask=re.compile(b"<<EJCMC.131, *(\w+)")

CFWeb = CF+".Web.tmp"
if os.path.isfile(CFWeb):
    with open(CFWeb,"rb") as f:
        ClDt = pickle.load(f)
        ClTs = pickle.load(f)
else:
    BaseR = urllib.request.urlopen(URL+RAW)
    BaseT = [b" ".join(reBase.findall(s)[0]).decode("utf").strip() for s in BaseR if reBase.match(s)]
    ClDt = "\n".join(BaseT)

    ClTs=""
    for s in ClDt.split('\n'):
        p, d = s.split()
        #PageR = urllib.request.urlopen(URL+"/"+p+RAW)
        time.sleep(1)
        for t in reTask.findall(urllib.request.urlopen(URL+"/"+p+RAW).read()):
            ClTs += "\n"+p+" "+t.decode("utf")
    print(ClDt)
    print(ClTs)
    with open(CFWeb,"wb") as f:
        pickle.dump(ClDt,f)
        pickle.dump(ClTs,f)

ClDt = {l:mkt(d[:4]+d[5:7]+d[8:10]+"000000") for l,d in (s.split() for s in ClDt.split("\n") if s)}
TsCl = {t:l for l,t in (s.split() for s in ClTs.split('\n') if s)}
TsDt = {t:ClDt[l] for t,l in TsCl.items()}

CFRes = CF+".All.tmp"
if os.path.isfile(CFRes):
    with open(CFRes,"rb") as f:
        Res = pickle.load(f)
else:
    import tarfile
    Res = table(set)
    with tarfile.open(CF,"r") as f:
        for o in f:
            if o.isfile():
                s = o.name
                # contest_86_20171219131616/000064-Nikscorp-HelloWorld-20170930113002.py
                ID, *Nick, Task, Date = s.split("/")[-1][:-3].split("-")
                Nick = "-".join(Nick)
                Res[Nick].add(Task)
    with open(CFRes,"wb") as f:
        pickle.dump(Res,f)
Users = {N for N,V in Res.items() if 3*len(V)>2*len(TsDt)}
print("# Users: {} total / {} allowed".format(len(Res), len(Users)))

CFSrc = CF+".Source.tmp"
if os.path.isfile(CFSrc):
    with open(CFSrc,"rb") as f:
        IDs = pickle.load(f)
        Src = pickle.load(f)
        Tab = pickle.load(f)
else:
    import tarfile
    Src = {}
    IDs = {}
    with tarfile.open(CF,"r") as f:
        for o in f:
            if o.isfile():
                s = o.name
                # contest_86_20171219131616/000064-Nikscorp-HelloWorld-20170930113002.py
                ID, *Nick, Task, Date = s.split("/")[-1][:-3].split("-")
                Nick, ID, Date = "-".join(Nick), int(ID), mkt(Date)
                if Nick not in Users: continue
                with f.extractfile(o) as df:
                    btxt = df.read()
                    try:
                        txt = btxt.decode()
                    except UnicodeDecodeError as E:
                        txt = btxt.decode("WINDOWS-1251")
                Src[ID], IDs[ID] = txt, (Nick, Task, Date)
    Tab = table(table_set)
    for ID, (Nick, Task, Date) in IDs.items():
        Tab[Task][Nick].add(ID)

    with open(CFSrc,"wb") as f:
        pickle.dump(IDs,f)
        pickle.dump(Src,f)
        pickle.dump(Tab,f)
print("# Tasks: {}/{}".format(len(IDs),max(IDs)))

def mktask(ID):
    src = Src[ID]
    prep = rb.sub(" ",ra.sub("@",ast.dump(ast.parse(src,"ex.py"),annotate_fields=False)))
    txt = autopep8.fix_code(src)
    for i,p in enumerate(AstK):
        prep = prep.replace(p,chr(i+0x21)+" ")
    return txt, prep.replace(" ","")

CFPrep = CF+".Prep.tmp"
if os.path.isfile(CFPrep):
    with open(CFPrep,"rb") as f:
        Prep = pickle.load(f)
else:
    import ast
    import re
    import autopep8
    import multiprocessing
    ra=re.compile(r"'[^']*'")
    rb=re.compile(r"[\[\]\{\}\(\)\,\ ]+")
    AstK = ['None ']+[s+' ' for s in dir(ast) if s[0].isalpha()]
    M = -1, int(max(IDs))
    pool = multiprocessing.Pool()
    S = sorted(Src)
    res = pool.map(mktask, S)
    #Prep = {}
    Prep = dict(zip(S, res))
    #for c, ID in enumerate(sorted(Src)):
    #    Prep[ID] = mktask(ID)
    #    if not c%100: print(c, end="\r")
    with open(CFPrep,"wb") as f:
        pickle.dump(Prep,f)
print("# Total code/prepared: {}/{}".format(sum(len(t) for t,p in Prep.values()), sum(len(p) for t,p in Prep.values())))

def getdist(id1, id2):
    return editdistance.eval(Prep[id1][1], Prep[id2][1])*2/(len(Prep[id1][1])+len(Prep[id2][1]))

def cluster(Heap, D="@"):
    l=-1
    while l!=len(Heap):
        l = len(Heap)
        Heap={frozenset.union(*(b for b in Heap if a&b)) for a in Heap}
    return Heap

def cluster2(I):
    Heap = { frozenset(c) for c in I.values() }
    return cluster(Heap)

def calctask(T):
    print("*", T)
    P, R = set(), table(set)
    Us = sorted(Tab[T])
    for i in range(len(Us)-1):
        for j in range(i+1,len(Us)):
            for ID1 in Tab[T][Us[i]]:
                R[ID1].add(ID1)
                for ID2 in Tab[T][Us[j]]:
                    ID1, ID2 = sorted((ID1, ID2))
                    dist = getdist(ID1, ID2)
                    if dist<cPaste:
                        P.add((ID1, ID2, dist))
                        R[ID1].add(ID2)
    C = cluster2(R)
    #C = cluster({ frozenset({ i, j }) for i, j, d in P })
    return P, C

CFPaste = CF+".Paste.tmp"
cPaste = 0.01
cRew = 0.1
minCommon = 7

if os.path.isfile(CFPaste):
    with open(CFPaste,"rb") as f:
        Paste = pickle.load(f)
else:
    import editdistance
    import multiprocessing
    pool = multiprocessing.Pool()
    res = pool.map(calctask, Tab)
    Paste = dict(zip(Tab, res))
    with open(CFPaste,"wb") as f:
        pickle.dump(Paste,f)

def tpasters(T):
    P = {}
    for C in Paste[T][1]:
        H, *L = sorted(C)
        U, L = IDs[H][0], {IDs[l][0] for l in L if IDs[l][0]!=IDs[H][0]}
        if not L: continue
        if len(L) >= minCommon:
            return None
        P[(U,H)] = L
    else:
        return P

def ftpasters(T):
    P = {}
    for C in Paste[T][1]:
        H, *L = sorted(C)
        if {IDs[l][0] for l in L} == {IDs[H][0]}:
            continue
        if len(L) >= minCommon:
            return {}
        P[(IDs[H][0],H)] = [(IDs[l][0],l) for l in L if IDs[l][0]!=IDs[H][0]]
    return P

def fpasters(T):
    P = {}
    for C in Paste[T][1]:
        H, *L = sorted(C)
        if {IDs[l][0] for l in L} == {IDs[H][0]}:
            continue
        P[(IDs[H][0],H)] = [(IDs[l][0],l) for l in L]
    return P

print("# Pasters:")

#{"HelloWorld","DummyClass", "NormalDouble","YieldFrom","SharedBrain","SectionShuffle","ParallelSegments","EvalFunction","CountInt","AndOr","DotsInCircle"}:

import difflib
import editdistance

def shortest(T,U1,U2):
    I1 = {int(U1)} if U1.isdigit() else Tab[T][U1]
    I2 = {int(U2)} if U2.isdigit() else Tab[T][U2]
    I1,I2 = min(((i1,i2) for i1 in I1 for i2 in I2), key=lambda a: getdist(*a))
    U1, U2 = IDs[I1][0], IDs[I2][0]
    return (I1,U1),(I2,U2),getdist(I1,I2)

def Allpasters():
    print(f"|||| '''{time.strftime('%F')}''' ||")
    for T in sorted(Paste):
        P = tpasters(T)
        print("|||| [[../Homework_{0}|{0}]]: {1}{2}||".format(T,sum(len(s) for s in Paste[T][1]),P and "." or "!"))
        if P and T not in TaskExs:
            for (U,H),L in sorted(P.items()):
                print("||{} ({}): || {} ||".format(U,H," ".join(L)))

def Taskpasters(T):
    for (U,H),L in fpasters(T).items():
        LL = " ".join("{}({})".format(*l) for l in L)
        print("{}({}): {}".format(U,H,LL))

def Taskids(T,U=None):
    UU = [U] if U else Tab[T]
    print("#  {}: {}".format(T, " ".join("{}({})".format(u,i) for i,u in sorted((i,IDs[i][0]) for U in UU for i in Tab[T][U]))))

def Userpastes(UU,TT=None):
    for T in ([TT] if TT else Paste):
        if not tpasters(T): continue
        for (U,H),L in fpasters(T).items():
            for u,i in L:
                if UU == u or UU == U:
                    print("{} {}: {}({}) → {}({})".format(T,getdist(H,i),U,H,u,i))

def Usertasks(U):
    print(" ".join(sorted(T for T in Tab if Tab[T][U])))

def Diffsrc(T,U1,U2):
    (I1,U1),(I2,U2),d = shortest(T,U1,U2)
    print("{} ({}) / {} ({}): {}\n".format(U1,I1,U2,I2,d),"\n".join(difflib.ndiff(Src[I1].splitlines(),Src[I2].splitlines())))

AuthIDs = set()
TaskExs = {"ChainSlice"}
def Mkpenalty(TaskEx=set(), addAUI=set()):
    Pen = {}
    AIDs = AuthIDs | addAUI
    for T in Tab:
        if T in TaskEx | TaskExs:
            Pen[T] = {}
            continue
        FP = ftpasters(T)
        #print(*(i for U,I in FP for u,i in FP[U,I] if u!=U))
        Pen[T]=set(i for U,I in FP for u,i in FP[U,I])
    return Pen
Penalty = Mkpenalty()

UP,MD,LW = 4,2,1
Div =  (24*60*60,UP), (7*24*60*60,MD), (14*24*60*60,LW),
def score(ID):
    U, T, D = IDs[ID]
    if ID in Penalty[T]: return 0
    for dd, sc in Div:
        if TsDt[T]+dd>=D:
            break
    return sc

def Userscore(U):
    scores = [max(score(i) for i in Tab[T][U]) for T in Tab if Tab[T][U]]
    All = len(scores)
    Full = scores.count(4)
    return sum(scores), All, Full

Mx = UP*len(Tab)
Mn = Mx * 2 // 3
def grade(M, g=("Отл","Хор","Хор","Удовл","Удовл")):
    if M<=Mn: return ""
    return g[int((Mx-M)*len(g)/(Mx-Mn))]

def isID(i):
    if type(i) is int and i in IDs or type(i) is str and i.isdigit() and int(i) in IDs:
        return int(i)
    else:
        return None

import os
import readline
import atexit

C = []

def dumpargs(*ap, **an):
    print("##",ap,an)

def completer(text, state):
    U = [u for u in Users if u.startswith(text)]
    return U[state] if len(U)>state else None

def lister(st, matches, maxlen):
    w = os.get_terminal_size().columns-2-len(prompt)
    sub = readline.get_line_buffer()
    res = " ".join(matches)[len(st):w+len(st)-1]
    print(" ".join(matches)[len(st):],prompt+sub,sep="\n",end="")
    sys.stdout.flush()

def DoCalc():
    if "/" in C:
        Mkpenalty(set(C[1:C.index("/")]),set(C[C.index("/")+1:]))
    else:
        Mkpenalty(set(C[1:]))

def DoTable():
    print(f'|||||||||||| {time.strftime("%F")} ||')
    print("|| № || '''Ник''' || '''Всего''' || '''Вовремя''' || '''Баллы''' || '''Оценка''' ||")
    Res = sorted(((Userscore(U),U) for U in Users),reverse=1)
    for i,((sc, a, f),U) in enumerate(Res,1):
        print("|| {} || {} || {} || {} || {} || {} ||".format(i,U,a,f,sc,grade(sc)))

def UserCalc():
    sc, a, f = Userscore(C[1])
    print("{}: ({}/{}) {} = {}".format(C[1],a,f,sc,grade(sc) or "Неуд"))

def DoTaskpasters():
    Taskpasters(C[0])

def DoUserpastes():
    if len(C)<2:
        Userpastes(C[0])
    else:
        Userpastes(C[1], C[0])

def ShowSource():
    print("{}({}):".format(*IDs[int(C[0])][:2]))
    print(Src[int(C[0])])

def DoDiffsrc():
    Diffsrc(*C)

def DoUsertasks():
    Usertasks(C[1])

def DoTaskids():
    if len(C)<3:
        Taskids(C[1])
    else:
        Taskids(C[1],C[2])

def DoUasage():
    print("Usage: {? [user] | ! [task] | id | user | task [user] | task user-or-id-1 user-or-id2}")

readline.set_completer(completer)
readline.set_completion_display_matches_hook(lister)

HFile =  CF+".history.tmp"
if os.path.isfile(HFile):
    readline.read_history_file(HFile)
atexit.register(readline.write_history_file, HFile)
cmd, prompt = "/", "GRR> "
while True:
    C[:] = cmd.split()
    if len(C)>0 and C[0]=="/":
        DoCalc()
    elif len(C)==1 and C[0]=="!":
        Allpasters()
    elif len(C)==1 and C[0]=="?":
        DoTable()
    elif len(C)==2 and C[0]=="?" and C[1] in Users:
        UserCalc()
    elif len(C)==1 and C[0] in Tab:
        DoTaskpasters()
    elif len(C)==1 and C[0] in Users:
        DoUserpastes()
    elif len(C)==2 and C[0] in Tab and C[1] in Users:
        DoUserpastes()
    elif len(C)==1 and isID(C[0]):
        ShowSource()
    elif len(C)==3 and C[0] in Tab:
        DoDiffsrc()
    elif len(C)==2 and C[0]=="!" and C[1] in Users:
        DoUsertasks()
    elif len(C)==2 and C[0]=="!" and C[1] in Tab:
        DoTaskids()
    elif len(C)==3 and C[0]=="!" and C[1] in Tab and C[2] in Users:
        DoTaskids()
    elif C:
        DoUasage()
    try:
        cmd = input(prompt)
    except EOFError:
        break