diff --git a/.flor/.replay.json b/.flor/.replay.json new file mode 100755 index 0000000..ebeca71 --- /dev/null +++ b/.flor/.replay.json @@ -0,0 +1,6 @@ +{ + "NAME": "CLRscheduler", + "TSTAMP": "2023-03-03T12:57:38.json", + "PROJID": "kaggle-nlp-disasters_flor.shadow", + "EPOCHS": 20 +} \ No newline at end of file diff --git a/.flor/log_records.csv b/.flor/log_records.csv new file mode 100644 index 0000000..1b43e87 --- /dev/null +++ b/.flor/log_records.csv @@ -0,0 +1,961 @@ +epoch,step,name,value +1,4,avg_train_loss,0.6317379772663116 +1,4,average_valid_loss,0.6872608090440432 +1,8,avg_train_loss,0.666976809501648 +1,8,average_valid_loss,0.6872601211071014 +1,12,avg_train_loss,0.6944272071123123 +1,12,average_valid_loss,0.6872410997748375 +1,16,avg_train_loss,0.6939918845891953 +1,16,average_valid_loss,0.6872148637970289 +1,20,avg_train_loss,0.7161572277545929 +1,20,average_valid_loss,0.6872355366746584 +1,24,avg_train_loss,0.6904194355010986 +1,24,average_valid_loss,0.6871495818098386 +1,28,avg_train_loss,0.7098240256309509 +1,28,average_valid_loss,0.6871780529618263 +1,32,avg_train_loss,0.7247538715600967 +1,32,average_valid_loss,0.6871854513883591 +1,36,avg_train_loss,0.6867499053478241 +1,36,average_valid_loss,0.6872002507249514 +1,40,avg_train_loss,0.6766587942838669 +1,40,average_valid_loss,0.6869727695981661 +1,44,avg_train_loss,0.687479555606842 +1,44,average_valid_loss,0.6868772109349569 +1,48,avg_train_loss,0.7056775987148285 +1,48,average_valid_loss,0.6869663596153259 +1,52,avg_train_loss,0.7045169770717621 +1,52,average_valid_loss,0.6873015314340591 +1,56,avg_train_loss,0.6911755949258804 +1,56,average_valid_loss,0.6874257897337278 +1,60,avg_train_loss,0.7044238299131393 +1,60,average_valid_loss,0.6875133266051611 +1,64,avg_train_loss,0.6972433179616928 +1,64,average_valid_loss,0.6876359060406685 +1,68,avg_train_loss,0.7074340432882309 +1,68,average_valid_loss,0.6884351919094721 +1,72,avg_train_loss,0.6778571307659149 +1,72,average_valid_loss,0.6876484826207161 +1,76,avg_train_loss,0.6971199661493301 +1,76,average_valid_loss,0.6876616254448891 +1,80,avg_train_loss,0.7003934681415558 +1,80,average_valid_loss,0.6878781542181969 +1,84,avg_train_loss,0.696274146437645 +1,84,average_valid_loss,0.6881575187047323 +1,88,avg_train_loss,0.6925397515296936 +1,88,average_valid_loss,0.6866878618796667 +1,92,avg_train_loss,0.673703745007515 +1,92,average_valid_loss,0.6855198889970779 +1,96,avg_train_loss,0.6440355330705643 +1,96,average_valid_loss,0.6852884987990061 +2,4,avg_train_loss,0.6005014479160309 +2,4,average_valid_loss,0.688323641816775 +2,8,avg_train_loss,0.6435199230909348 +2,8,average_valid_loss,0.689600924650828 +2,12,avg_train_loss,0.6888653039932251 +2,12,average_valid_loss,0.687987819314003 +2,16,avg_train_loss,0.6976613402366638 +2,16,average_valid_loss,0.6864538465936979 +2,20,avg_train_loss,0.7287068665027618 +2,20,average_valid_loss,0.6843054840962092 +2,24,avg_train_loss,0.679160088300705 +2,24,average_valid_loss,0.684039461115996 +2,28,avg_train_loss,0.6994520723819733 +2,28,average_valid_loss,0.6835709686080614 +2,32,avg_train_loss,0.7077995240688324 +2,32,average_valid_loss,0.6835813671350479 +2,36,avg_train_loss,0.6902900785207748 +2,36,average_valid_loss,0.6834816311796507 +2,40,avg_train_loss,0.6785358786582947 +2,40,average_valid_loss,0.6829317733645439 +2,44,avg_train_loss,0.6928768455982208 +2,44,average_valid_loss,0.6827298253774643 +2,48,avg_train_loss,0.7018976658582687 +2,48,average_valid_loss,0.6832175329327583 +2,52,avg_train_loss,0.705698236823082 +2,52,average_valid_loss,0.6841531917452812 +2,56,avg_train_loss,0.6900441497564316 +2,56,average_valid_loss,0.6842819626132647 +2,60,avg_train_loss,0.695109099149704 +2,60,average_valid_loss,0.6848881617188454 +2,64,avg_train_loss,0.6942741423845291 +2,64,average_valid_loss,0.6854944999019305 +2,68,avg_train_loss,0.7007605582475662 +2,68,average_valid_loss,0.6876463169852892 +2,72,avg_train_loss,0.6851154565811157 +2,72,average_valid_loss,0.6848103528221449 +2,76,avg_train_loss,0.6930024474859238 +2,76,average_valid_loss,0.6840579037865003 +2,80,avg_train_loss,0.6915064454078674 +2,80,average_valid_loss,0.6845764443278313 +2,84,avg_train_loss,0.6875742375850677 +2,84,average_valid_loss,0.6849800075093905 +2,88,avg_train_loss,0.6817505061626434 +2,88,average_valid_loss,0.6830458839734396 +2,92,avg_train_loss,0.6600783467292786 +2,92,average_valid_loss,0.6820869644482931 +2,96,avg_train_loss,0.6140369772911072 +2,96,average_valid_loss,0.6858095998565356 +3,4,avg_train_loss,0.5665127635002136 +3,4,average_valid_loss,0.6970185289780298 +3,8,avg_train_loss,0.6363857537508011 +3,8,average_valid_loss,0.6970967476566633 +3,12,avg_train_loss,0.691016897559166 +3,12,average_valid_loss,0.6910599420468012 +3,16,avg_train_loss,0.697862058877945 +3,16,average_valid_loss,0.6861181706190109 +3,20,avg_train_loss,0.7233260273933411 +3,20,average_valid_loss,0.6813119997580847 +3,24,avg_train_loss,0.6756205558776855 +3,24,average_valid_loss,0.681013710796833 +3,28,avg_train_loss,0.7003984153270721 +3,28,average_valid_loss,0.6806494121750196 +3,32,avg_train_loss,0.69867043197155 +3,32,average_valid_loss,0.6815921664237976 +3,36,avg_train_loss,0.6882121562957764 +3,36,average_valid_loss,0.6811891918381056 +3,40,avg_train_loss,0.6664448827505112 +3,40,average_valid_loss,0.6799168984095255 +3,44,avg_train_loss,0.682912141084671 +3,44,average_valid_loss,0.6800147444009781 +3,48,avg_train_loss,0.701073631644249 +3,48,average_valid_loss,0.6807589903473854 +3,52,avg_train_loss,0.699536144733429 +3,52,average_valid_loss,0.6825462132692337 +3,56,avg_train_loss,0.6884040832519531 +3,56,average_valid_loss,0.6819929977258047 +3,60,avg_train_loss,0.6941366940736771 +3,60,average_valid_loss,0.6825986976424853 +3,64,avg_train_loss,0.6943193972110748 +3,64,average_valid_loss,0.6838163112600645 +3,68,avg_train_loss,0.6961643546819687 +3,68,average_valid_loss,0.6872540960709254 +3,72,avg_train_loss,0.683580681681633 +3,72,average_valid_loss,0.6823013300697008 +3,76,avg_train_loss,0.6875761449337006 +3,76,average_valid_loss,0.6819501395026842 +3,80,avg_train_loss,0.692571684718132 +3,80,average_valid_loss,0.6825394382079443 +3,84,avg_train_loss,0.6867320686578751 +3,84,average_valid_loss,0.6819517935315768 +3,88,avg_train_loss,0.6846296787261963 +3,88,average_valid_loss,0.6804749444127083 +3,92,avg_train_loss,0.6572228670120239 +3,92,average_valid_loss,0.679274765153726 +3,96,avg_train_loss,0.6135052293539047 +3,96,average_valid_loss,0.6876237243413925 +4,4,avg_train_loss,0.5608552992343903 +4,4,average_valid_loss,0.7057649518052737 +4,8,avg_train_loss,0.6313475966453552 +4,8,average_valid_loss,0.7020106986165047 +4,12,avg_train_loss,0.6887009888887405 +4,12,average_valid_loss,0.6888291810949644 +4,16,avg_train_loss,0.6971220076084137 +4,16,average_valid_loss,0.6826933224995931 +4,20,avg_train_loss,0.7019226849079132 +4,20,average_valid_loss,0.6778182884057363 +4,24,avg_train_loss,0.67058365046978 +4,24,average_valid_loss,0.6775213678677877 +4,28,avg_train_loss,0.6973124891519547 +4,28,average_valid_loss,0.6779876376191775 +4,32,avg_train_loss,0.6985982805490494 +4,32,average_valid_loss,0.679691898326079 +4,36,avg_train_loss,0.6905513852834702 +4,36,average_valid_loss,0.6785931165019671 +4,40,avg_train_loss,0.6625869870185852 +4,40,average_valid_loss,0.6771674255530039 +4,44,avg_train_loss,0.6814477890729904 +4,44,average_valid_loss,0.6770887424548467 +4,48,avg_train_loss,0.7045919597148895 +4,48,average_valid_loss,0.6776558309793472 +4,52,avg_train_loss,0.6935695111751556 +4,52,average_valid_loss,0.6809701894720396 +4,56,avg_train_loss,0.6887829452753067 +4,56,average_valid_loss,0.6796489184101423 +4,60,avg_train_loss,0.6833103895187378 +4,60,average_valid_loss,0.6798736279209455 +4,64,avg_train_loss,0.6980576366186142 +4,64,average_valid_loss,0.6823601101835569 +4,68,avg_train_loss,0.6941166669130325 +4,68,average_valid_loss,0.6893649424115816 +4,72,avg_train_loss,0.6912165135145187 +4,72,average_valid_loss,0.6803880458076795 +4,76,avg_train_loss,0.6915243715047836 +4,76,average_valid_loss,0.6807131469249725 +4,80,avg_train_loss,0.6913430392742157 +4,80,average_valid_loss,0.6812107240160307 +4,84,avg_train_loss,0.6898874640464783 +4,84,average_valid_loss,0.6806210254629453 +4,88,avg_train_loss,0.6852130740880966 +4,88,average_valid_loss,0.6773185084263483 +4,92,avg_train_loss,0.6491264998912811 +4,92,average_valid_loss,0.6779796555638313 +4,96,avg_train_loss,0.5911271721124649 +4,96,average_valid_loss,0.6918621261914571 +5,4,avg_train_loss,0.5492565333843231 +5,4,average_valid_loss,0.7156848659118017 +5,8,avg_train_loss,0.6339734345674515 +5,8,average_valid_loss,0.7048475816845894 +5,12,avg_train_loss,0.6887326240539551 +5,12,average_valid_loss,0.6878956481814384 +5,16,avg_train_loss,0.6963336020708084 +5,16,average_valid_loss,0.6804408406217893 +5,20,avg_train_loss,0.7040726840496063 +5,20,average_valid_loss,0.6754432817300161 +5,24,avg_train_loss,0.6718475967645645 +5,24,average_valid_loss,0.6754380514224371 +5,28,avg_train_loss,0.6878447234630585 +5,28,average_valid_loss,0.675753449400266 +5,32,avg_train_loss,0.6925942599773407 +5,32,average_valid_loss,0.678731806576252 +5,36,avg_train_loss,0.68504299223423 +5,36,average_valid_loss,0.6758468151092529 +5,40,avg_train_loss,0.6752417385578156 +5,40,average_valid_loss,0.6740845665335655 +5,44,avg_train_loss,0.6763578802347183 +5,44,average_valid_loss,0.674012765288353 +5,48,avg_train_loss,0.7009853720664978 +5,48,average_valid_loss,0.6756170094013214 +5,52,avg_train_loss,0.6947735399007797 +5,52,average_valid_loss,0.6790984471638998 +5,56,avg_train_loss,0.6858357489109039 +5,56,average_valid_loss,0.6775634462634722 +5,60,avg_train_loss,0.6955967396497726 +5,60,average_valid_loss,0.6796500757336617 +5,64,avg_train_loss,0.6862735897302628 +5,64,average_valid_loss,0.6834694097439448 +5,68,avg_train_loss,0.6934970915317535 +5,68,average_valid_loss,0.6888041297594706 +5,72,avg_train_loss,0.6877834051847458 +5,72,average_valid_loss,0.6760392983754476 +5,76,avg_train_loss,0.6969411075115204 +5,76,average_valid_loss,0.6771780252456665 +5,80,avg_train_loss,0.6882379651069641 +5,80,average_valid_loss,0.6785922596851984 +5,84,avg_train_loss,0.689894288778305 +5,84,average_valid_loss,0.6786707838376363 +5,88,avg_train_loss,0.6833094954490662 +5,88,average_valid_loss,0.6748639047145844 +5,92,avg_train_loss,0.6483035534620285 +5,92,average_valid_loss,0.6760684450467428 +5,96,avg_train_loss,0.6189875900745392 +5,96,average_valid_loss,0.6907699803511301 +6,4,avg_train_loss,0.5361777544021606 +6,4,average_valid_loss,0.7184828668832779 +6,8,avg_train_loss,0.6414842158555984 +6,8,average_valid_loss,0.7050631120800972 +6,12,avg_train_loss,0.6879852563142776 +6,12,average_valid_loss,0.6857672010858854 +6,16,avg_train_loss,0.6953363418579102 +6,16,average_valid_loss,0.6784335871537527 +6,20,avg_train_loss,0.694896399974823 +6,20,average_valid_loss,0.6760734344522158 +6,24,avg_train_loss,0.676605835556984 +6,24,average_valid_loss,0.6735352724790573 +6,28,avg_train_loss,0.6972926557064056 +6,28,average_valid_loss,0.675167016685009 +6,32,avg_train_loss,0.6814057677984238 +6,32,average_valid_loss,0.6773223032553991 +6,36,avg_train_loss,0.6992046386003494 +6,36,average_valid_loss,0.6742339159051577 +6,40,avg_train_loss,0.6613060086965561 +6,40,average_valid_loss,0.6744737103581429 +6,44,avg_train_loss,0.686052143573761 +6,44,average_valid_loss,0.6742869317531586 +6,48,avg_train_loss,0.7038759738206863 +6,48,average_valid_loss,0.6741989105939865 +6,52,avg_train_loss,0.6949267387390137 +6,52,average_valid_loss,0.6781431883573532 +6,56,avg_train_loss,0.6907156705856323 +6,56,average_valid_loss,0.6768019249041876 +6,60,avg_train_loss,0.6827747672796249 +6,60,average_valid_loss,0.6768641571203867 +6,64,avg_train_loss,0.6883632689714432 +6,64,average_valid_loss,0.6791180074214935 +6,68,avg_train_loss,0.6978093981742859 +6,68,average_valid_loss,0.687746619184812 +6,72,avg_train_loss,0.6870283633470535 +6,72,average_valid_loss,0.6750478744506836 +6,76,avg_train_loss,0.6853098720312119 +6,76,average_valid_loss,0.675813282529513 +6,80,avg_train_loss,0.6941485106945038 +6,80,average_valid_loss,0.6775150249401728 +6,84,avg_train_loss,0.6899810284376144 +6,84,average_valid_loss,0.6786124333739281 +6,88,avg_train_loss,0.6767929941415787 +6,88,average_valid_loss,0.6747976119319598 +6,92,avg_train_loss,0.6563175767660141 +6,92,average_valid_loss,0.6776261478662491 +6,96,avg_train_loss,0.6026453226804733 +6,96,average_valid_loss,0.6995255053043365 +7,4,avg_train_loss,0.5242820307612419 +7,4,average_valid_loss,0.7346215024590492 +7,8,avg_train_loss,0.6407649666070938 +7,8,average_valid_loss,0.7064430887500445 +7,12,avg_train_loss,0.692553386092186 +7,12,average_valid_loss,0.6857660437623659 +7,16,avg_train_loss,0.6929579824209213 +7,16,average_valid_loss,0.676415520409743 +7,20,avg_train_loss,0.6910476237535477 +7,20,average_valid_loss,0.6752679472168287 +7,24,avg_train_loss,0.6705624908208847 +7,24,average_valid_loss,0.6724252452452978 +7,28,avg_train_loss,0.6888911724090576 +7,28,average_valid_loss,0.6747610891858736 +7,32,avg_train_loss,0.6989502608776093 +7,32,average_valid_loss,0.6779491901397705 +7,36,avg_train_loss,0.6835406869649887 +7,36,average_valid_loss,0.6723266616463661 +7,40,avg_train_loss,0.6630026549100876 +7,40,average_valid_loss,0.6730381747086843 +7,44,avg_train_loss,0.6754554957151413 +7,44,average_valid_loss,0.6714052086075147 +7,48,avg_train_loss,0.7007889598608017 +7,48,average_valid_loss,0.6724637821316719 +7,52,avg_train_loss,0.6894766688346863 +7,52,average_valid_loss,0.6786858811974525 +7,56,avg_train_loss,0.6976516246795654 +7,56,average_valid_loss,0.6763240694999695 +7,60,avg_train_loss,0.6838807612657547 +7,60,average_valid_loss,0.675138254960378 +7,64,avg_train_loss,0.6861367970705032 +7,64,average_valid_loss,0.6776760195692381 +7,68,avg_train_loss,0.6808171421289444 +7,68,average_valid_loss,0.6854997649788857 +7,72,avg_train_loss,0.6969883292913437 +7,72,average_valid_loss,0.672934003174305 +7,76,avg_train_loss,0.6858567595481873 +7,76,average_valid_loss,0.673940509557724 +7,80,avg_train_loss,0.6919941604137421 +7,80,average_valid_loss,0.6748988678057989 +7,84,avg_train_loss,0.6814596652984619 +7,84,average_valid_loss,0.6772356977065405 +7,88,avg_train_loss,0.6814961433410645 +7,88,average_valid_loss,0.6734452272454897 +7,92,avg_train_loss,0.6480002701282501 +7,92,average_valid_loss,0.6748681589961052 +7,96,avg_train_loss,0.5941737145185471 +7,96,average_valid_loss,0.6996975839138031 +8,4,avg_train_loss,0.5344903320074081 +8,4,average_valid_loss,0.7347570980588595 +8,8,avg_train_loss,0.6520051956176758 +8,8,average_valid_loss,0.7031970024108887 +8,12,avg_train_loss,0.690180629491806 +8,12,average_valid_loss,0.6816223586599032 +8,16,avg_train_loss,0.6932737082242966 +8,16,average_valid_loss,0.6756524170438448 +8,20,avg_train_loss,0.6952943503856659 +8,20,average_valid_loss,0.6764378100633621 +8,24,avg_train_loss,0.6670368313789368 +8,24,average_valid_loss,0.6715935170650482 +8,28,avg_train_loss,0.6856862902641296 +8,28,average_valid_loss,0.6741219311952591 +8,32,avg_train_loss,0.6915982514619827 +8,32,average_valid_loss,0.6771677508950233 +8,36,avg_train_loss,0.6943055540323257 +8,36,average_valid_loss,0.6724518612027168 +8,40,avg_train_loss,0.6540458351373672 +8,40,average_valid_loss,0.6716668556133906 +8,44,avg_train_loss,0.6939978450536728 +8,44,average_valid_loss,0.67240259796381 +8,48,avg_train_loss,0.6893952190876007 +8,48,average_valid_loss,0.672874261935552 +8,52,avg_train_loss,0.6816552877426147 +8,52,average_valid_loss,0.6798464705546697 +8,56,avg_train_loss,0.6937831789255142 +8,56,average_valid_loss,0.6749873384833336 +8,60,avg_train_loss,0.6823574900627136 +8,60,average_valid_loss,0.6737751290202141 +8,64,avg_train_loss,0.6953515410423279 +8,64,average_valid_loss,0.6772379303971926 +8,68,avg_train_loss,0.693393275141716 +8,68,average_valid_loss,0.6889758581916491 +8,72,avg_train_loss,0.6739634126424789 +8,72,average_valid_loss,0.6723704114556313 +8,76,avg_train_loss,0.696665495634079 +8,76,average_valid_loss,0.6742646023631096 +8,80,avg_train_loss,0.6863385885953903 +8,80,average_valid_loss,0.6755628064274788 +8,84,avg_train_loss,0.6892296224832535 +8,84,average_valid_loss,0.6783633505304655 +8,88,avg_train_loss,0.690071314573288 +8,88,average_valid_loss,0.673483761648337 +8,92,avg_train_loss,0.6446410715579987 +8,92,average_valid_loss,0.6747577711939812 +8,96,avg_train_loss,0.5903660878539085 +8,96,average_valid_loss,0.7076633274555206 +9,4,avg_train_loss,0.5063857883214951 +9,4,average_valid_loss,0.7441073507070541 +9,8,avg_train_loss,0.6336732655763626 +9,8,average_valid_loss,0.6999517604708672 +9,12,avg_train_loss,0.6763178259134293 +9,12,average_valid_loss,0.6785130376617113 +9,16,avg_train_loss,0.6895873099565506 +9,16,average_valid_loss,0.6717338636517525 +9,20,avg_train_loss,0.6865237355232239 +9,20,average_valid_loss,0.6730640927950541 +9,24,avg_train_loss,0.6734685748815536 +9,24,average_valid_loss,0.6686747372150421 +9,28,avg_train_loss,0.6815193593502045 +9,28,average_valid_loss,0.672155849635601 +9,32,avg_train_loss,0.7048343122005463 +9,32,average_valid_loss,0.6757262100776037 +9,36,avg_train_loss,0.683683305978775 +9,36,average_valid_loss,0.6701504662632942 +9,40,avg_train_loss,0.662037268280983 +9,40,average_valid_loss,0.6723606362938881 +9,44,avg_train_loss,0.6784507483243942 +9,44,average_valid_loss,0.6704457923769951 +9,48,avg_train_loss,0.7131697535514832 +9,48,average_valid_loss,0.6729415183266004 +9,52,avg_train_loss,0.6813401877880096 +9,52,average_valid_loss,0.6807522227366766 +9,56,avg_train_loss,0.6911351084709167 +9,56,average_valid_loss,0.6754439746340116 +9,60,avg_train_loss,0.682423859834671 +9,60,average_valid_loss,0.6732939357558886 +9,64,avg_train_loss,0.6936230361461639 +9,64,average_valid_loss,0.6792892441153526 +9,68,avg_train_loss,0.677094966173172 +9,68,average_valid_loss,0.6953361655275027 +9,72,avg_train_loss,0.6907272785902023 +9,72,average_valid_loss,0.6708092366655668 +9,76,avg_train_loss,0.6936379820108414 +9,76,average_valid_loss,0.6726650396982828 +9,80,avg_train_loss,0.6835174709558487 +9,80,average_valid_loss,0.6739047691226006 +9,84,avg_train_loss,0.6899255365133286 +9,84,average_valid_loss,0.6770056858658791 +9,88,avg_train_loss,0.6777763217687607 +9,88,average_valid_loss,0.6715724021196365 +9,92,avg_train_loss,0.6590285152196884 +9,92,average_valid_loss,0.6751684273282686 +9,96,avg_train_loss,0.5755433067679405 +9,96,average_valid_loss,0.7077464088797569 +10,4,avg_train_loss,0.5152088180184364 +10,4,average_valid_loss,0.7426493242383003 +10,8,avg_train_loss,0.6297043859958649 +10,8,average_valid_loss,0.6986995538075765 +10,12,avg_train_loss,0.6752003878355026 +10,12,average_valid_loss,0.6747341553370158 +10,16,avg_train_loss,0.6883811205625534 +10,16,average_valid_loss,0.6698982988794645 +10,20,avg_train_loss,0.6810365170240402 +10,20,average_valid_loss,0.6698419054349264 +10,24,avg_train_loss,0.674247220158577 +10,24,average_valid_loss,0.6683158228794733 +10,28,avg_train_loss,0.6794648915529251 +10,28,average_valid_loss,0.6726408203442892 +10,32,avg_train_loss,0.6851761788129807 +10,32,average_valid_loss,0.6748525227109591 +10,36,avg_train_loss,0.6858533322811127 +10,36,average_valid_loss,0.6667364736398061 +10,40,avg_train_loss,0.6671319752931595 +10,40,average_valid_loss,0.671202041208744 +10,44,avg_train_loss,0.6704376339912415 +10,44,average_valid_loss,0.6677309547861418 +10,48,avg_train_loss,0.6886996328830719 +10,48,average_valid_loss,0.6710313061873118 +10,52,avg_train_loss,0.6976743638515472 +10,52,average_valid_loss,0.6792106255888939 +10,56,avg_train_loss,0.6849856972694397 +10,56,average_valid_loss,0.6750449438889822 +10,60,avg_train_loss,0.6698623448610306 +10,60,average_valid_loss,0.6715250462293625 +10,64,avg_train_loss,0.6865374892950058 +10,64,average_valid_loss,0.6759836028019587 +10,68,avg_train_loss,0.6959231793880463 +10,68,average_valid_loss,0.6882902209957441 +10,72,avg_train_loss,0.6873447000980377 +10,72,average_valid_loss,0.6690453340609869 +10,76,avg_train_loss,0.7074000388383865 +10,76,average_valid_loss,0.6722464685638746 +10,80,avg_train_loss,0.6766635328531265 +10,80,average_valid_loss,0.6747033099333445 +10,84,avg_train_loss,0.6839406490325928 +10,84,average_valid_loss,0.6781802326440811 +10,88,avg_train_loss,0.6917552649974823 +10,88,average_valid_loss,0.672417531410853 +10,92,avg_train_loss,0.6444031745195389 +10,92,average_valid_loss,0.6743702093760172 +10,96,avg_train_loss,0.5929523557424545 +10,96,average_valid_loss,0.707210453848044 +11,4,avg_train_loss,0.521454282104969 +11,4,average_valid_loss,0.7454331666231155 +11,8,avg_train_loss,0.6385126411914825 +11,8,average_valid_loss,0.7002092103163401 +11,12,avg_train_loss,0.680075615644455 +11,12,average_valid_loss,0.6760607386628786 +11,16,avg_train_loss,0.6843600869178772 +11,16,average_valid_loss,0.6715315828720728 +11,20,avg_train_loss,0.6852649003267288 +11,20,average_valid_loss,0.6718239511052767 +11,24,avg_train_loss,0.683311402797699 +11,24,average_valid_loss,0.6691968739032745 +11,28,avg_train_loss,0.6818951666355133 +11,28,average_valid_loss,0.6718999619285265 +11,32,avg_train_loss,0.6924597024917603 +11,32,average_valid_loss,0.6771830295523008 +11,36,avg_train_loss,0.6883558928966522 +11,36,average_valid_loss,0.6697093571225802 +11,40,avg_train_loss,0.6522798240184784 +11,40,average_valid_loss,0.6715692728757858 +11,44,avg_train_loss,0.6706527918577194 +11,44,average_valid_loss,0.6681411092480024 +11,48,avg_train_loss,0.7100192755460739 +11,48,average_valid_loss,0.6712794502576193 +11,52,avg_train_loss,0.6874718219041824 +11,52,average_valid_loss,0.6784574041763941 +11,56,avg_train_loss,0.6953529715538025 +11,56,average_valid_loss,0.6764396404226621 +11,60,avg_train_loss,0.6902159452438354 +11,60,average_valid_loss,0.6755096564690272 +11,64,avg_train_loss,0.6867004632949829 +11,64,average_valid_loss,0.6769816726446152 +11,68,avg_train_loss,0.6978352814912796 +11,68,average_valid_loss,0.6917775322993597 +11,72,avg_train_loss,0.6822302937507629 +11,72,average_valid_loss,0.668345496058464 +11,76,avg_train_loss,0.6838574111461639 +11,76,average_valid_loss,0.6702767387032509 +11,80,avg_train_loss,0.688614159822464 +11,80,average_valid_loss,0.6717506175239881 +11,84,avg_train_loss,0.685509204864502 +11,84,average_valid_loss,0.6750768174727758 +11,88,avg_train_loss,0.6873057633638382 +11,88,average_valid_loss,0.6705459331472715 +11,92,avg_train_loss,0.6504253298044205 +11,92,average_valid_loss,0.6732938786347707 +11,96,avg_train_loss,0.5858976691961288 +11,96,average_valid_loss,0.7144283826152483 +12,4,avg_train_loss,0.505040243268013 +12,4,average_valid_loss,0.7514288648962975 +12,8,avg_train_loss,0.6421871334314346 +12,8,average_valid_loss,0.6934239367643992 +12,12,avg_train_loss,0.68889319896698 +12,12,average_valid_loss,0.6761341169476509 +12,16,avg_train_loss,0.6897916793823242 +12,16,average_valid_loss,0.6715835382541021 +12,20,avg_train_loss,0.6803017854690552 +12,20,average_valid_loss,0.67323865989844 +12,24,avg_train_loss,0.6744921058416367 +12,24,average_valid_loss,0.6687001685301462 +12,28,avg_train_loss,0.6792099326848984 +12,28,average_valid_loss,0.6715424607197443 +12,32,avg_train_loss,0.679956704378128 +12,32,average_valid_loss,0.6736143877108892 +12,36,avg_train_loss,0.6766994744539261 +12,36,average_valid_loss,0.6658497825264931 +12,40,avg_train_loss,0.6608720272779465 +12,40,average_valid_loss,0.669233667353789 +12,44,avg_train_loss,0.6712822467088699 +12,44,average_valid_loss,0.6668610572814941 +12,48,avg_train_loss,0.7114253640174866 +12,48,average_valid_loss,0.6694012607137362 +12,52,avg_train_loss,0.6910409778356552 +12,52,average_valid_loss,0.6802239542206129 +12,56,avg_train_loss,0.677278459072113 +12,56,average_valid_loss,0.6741739188631376 +12,60,avg_train_loss,0.6872082352638245 +12,60,average_valid_loss,0.6714094231526057 +12,64,avg_train_loss,0.6967447847127914 +12,64,average_valid_loss,0.6776399314403534 +12,68,avg_train_loss,0.6829631179571152 +12,68,average_valid_loss,0.6984961926937103 +12,72,avg_train_loss,0.6855712682008743 +12,72,average_valid_loss,0.6676767095923424 +12,76,avg_train_loss,0.6947779357433319 +12,76,average_valid_loss,0.669121210773786 +12,80,avg_train_loss,0.6931906491518021 +12,80,average_valid_loss,0.6715046490232149 +12,84,avg_train_loss,0.6918817609548569 +12,84,average_valid_loss,0.6791243329644203 +12,88,avg_train_loss,0.6827407628297806 +12,88,average_valid_loss,0.6709557597835859 +12,92,avg_train_loss,0.649899959564209 +12,92,average_valid_loss,0.6744134897987047 +12,96,avg_train_loss,0.6008869856595993 +12,96,average_valid_loss,0.7197801147898039 +13,4,avg_train_loss,0.5062768086791039 +13,4,average_valid_loss,0.7547587131460508 +13,8,avg_train_loss,0.6430866420269012 +13,8,average_valid_loss,0.6938665310541788 +13,12,avg_train_loss,0.6807112544775009 +13,12,average_valid_loss,0.6746775930126508 +13,16,avg_train_loss,0.6800292581319809 +13,16,average_valid_loss,0.6706260840098063 +13,20,avg_train_loss,0.6887939423322678 +13,20,average_valid_loss,0.6738914872209231 +13,24,avg_train_loss,0.6630164682865143 +13,24,average_valid_loss,0.6667683497071266 +13,28,avg_train_loss,0.6829326003789902 +13,28,average_valid_loss,0.6692663605014483 +13,32,avg_train_loss,0.6995691508054733 +13,32,average_valid_loss,0.6771077762047449 +13,36,avg_train_loss,0.6921916007995605 +13,36,average_valid_loss,0.6683742652336756 +13,40,avg_train_loss,0.6585580110549927 +13,40,average_valid_loss,0.6710944374402364 +13,44,avg_train_loss,0.6813062429428101 +13,44,average_valid_loss,0.6685610115528107 +13,48,avg_train_loss,0.7018939852714539 +13,48,average_valid_loss,0.6721377124389013 +13,52,avg_train_loss,0.690077617764473 +13,52,average_valid_loss,0.6841533506910006 +13,56,avg_train_loss,0.6874997913837433 +13,56,average_valid_loss,0.6755842044949532 +13,60,avg_train_loss,0.6777959764003754 +13,60,average_valid_loss,0.6738993525505066 +13,64,avg_train_loss,0.6857629865407944 +13,64,average_valid_loss,0.677996593217055 +13,68,avg_train_loss,0.6898452192544937 +13,68,average_valid_loss,0.6932665904362997 +13,72,avg_train_loss,0.6895801573991776 +13,72,average_valid_loss,0.6678109963734945 +13,76,avg_train_loss,0.6894680112600327 +13,76,average_valid_loss,0.6702169552445412 +13,80,avg_train_loss,0.6798747628927231 +13,80,average_valid_loss,0.6714801962176958 +13,84,avg_train_loss,0.6905244886875153 +13,84,average_valid_loss,0.6780388777454694 +13,88,avg_train_loss,0.68928062915802 +13,88,average_valid_loss,0.6717801814277967 +13,92,avg_train_loss,0.6491261124610901 +13,92,average_valid_loss,0.6741530199845632 +13,96,avg_train_loss,0.5767435878515244 +13,96,average_valid_loss,0.7194677939017614 +14,4,avg_train_loss,0.5082144886255264 +14,4,average_valid_loss,0.7557030394673347 +14,8,avg_train_loss,0.6411832123994827 +14,8,average_valid_loss,0.6875196695327759 +14,12,avg_train_loss,0.6740975230932236 +14,12,average_valid_loss,0.6709153950214386 +14,16,avg_train_loss,0.6839145421981812 +14,16,average_valid_loss,0.6684959158301353 +14,20,avg_train_loss,0.6878928989171982 +14,20,average_valid_loss,0.6684962784250578 +14,24,avg_train_loss,0.6624243706464767 +14,24,average_valid_loss,0.6672311872243881 +14,28,avg_train_loss,0.6816137582063675 +14,28,average_valid_loss,0.6709597880641619 +14,32,avg_train_loss,0.7006447017192841 +14,32,average_valid_loss,0.671194759507974 +14,36,avg_train_loss,0.6705610454082489 +14,36,average_valid_loss,0.6643030171593031 +14,40,avg_train_loss,0.6581270694732666 +14,40,average_valid_loss,0.6680463775992393 +14,44,avg_train_loss,0.6824835240840912 +14,44,average_valid_loss,0.6651181926329931 +14,48,avg_train_loss,0.7124291807413101 +14,48,average_valid_loss,0.669763778646787 +14,52,avg_train_loss,0.6768581718206406 +14,52,average_valid_loss,0.6824750478068987 +14,56,avg_train_loss,0.6869267374277115 +14,56,average_valid_loss,0.6733082582553228 +14,60,avg_train_loss,0.677844911813736 +14,60,average_valid_loss,0.6671849315365156 +14,64,avg_train_loss,0.6847217231988907 +14,64,average_valid_loss,0.6713951999942461 +14,68,avg_train_loss,0.6824377626180649 +14,68,average_valid_loss,0.7018021369973818 +14,72,avg_train_loss,0.6847259998321533 +14,72,average_valid_loss,0.6644417146841685 +14,76,avg_train_loss,0.6856972277164459 +14,76,average_valid_loss,0.666109157105287 +14,80,avg_train_loss,0.6892751753330231 +14,80,average_valid_loss,0.6699948857227961 +14,84,avg_train_loss,0.6877083480358124 +14,84,average_valid_loss,0.6765351121624311 +14,88,avg_train_loss,0.7075381577014923 +14,88,average_valid_loss,0.6710532233119011 +14,92,avg_train_loss,0.6509678959846497 +14,92,average_valid_loss,0.6712431758642197 +14,96,avg_train_loss,0.5936902314424515 +14,96,average_valid_loss,0.7380170052250227 +15,4,avg_train_loss,0.5045086368918419 +15,4,average_valid_loss,0.7585752060015997 +15,8,avg_train_loss,0.6429192423820496 +15,8,average_valid_loss,0.6886877864599228 +15,12,avg_train_loss,0.6727182865142822 +15,12,average_valid_loss,0.6749305501580238 +15,16,avg_train_loss,0.6796796917915344 +15,16,average_valid_loss,0.6693440973758698 +15,20,avg_train_loss,0.6904970407485962 +15,20,average_valid_loss,0.6725700944662094 +15,24,avg_train_loss,0.6688915938138962 +15,24,average_valid_loss,0.6663704613844553 +15,28,avg_train_loss,0.6839903593063354 +15,28,average_valid_loss,0.6678759008646011 +15,32,avg_train_loss,0.6910656690597534 +15,32,average_valid_loss,0.6712405309081078 +15,36,avg_train_loss,0.6839850693941116 +15,36,average_valid_loss,0.6653071120381355 +15,40,avg_train_loss,0.6452499777078629 +15,40,average_valid_loss,0.6672559653719267 +15,44,avg_train_loss,0.6778631657361984 +15,44,average_valid_loss,0.6648137321074804 +15,48,avg_train_loss,0.7018586695194244 +15,48,average_valid_loss,0.6687842582662901 +15,52,avg_train_loss,0.6918921172618866 +15,52,average_valid_loss,0.6866628030935923 +15,56,avg_train_loss,0.6874098777770996 +15,56,average_valid_loss,0.6719031582276026 +15,60,avg_train_loss,0.672059953212738 +15,60,average_valid_loss,0.6667478059728941 +15,64,avg_train_loss,0.690949872136116 +15,64,average_valid_loss,0.6764932249983152 +15,68,avg_train_loss,0.6850815266370773 +15,68,average_valid_loss,0.6943544968962669 +15,72,avg_train_loss,0.6979095637798309 +15,72,average_valid_loss,0.6650446777542433 +15,76,avg_train_loss,0.6925019770860672 +15,76,average_valid_loss,0.6687012935678164 +15,80,avg_train_loss,0.6854312866926193 +15,80,average_valid_loss,0.6701361214121183 +15,84,avg_train_loss,0.6787927448749542 +15,84,average_valid_loss,0.6788778578241667 +15,88,avg_train_loss,0.6895885616540909 +15,88,average_valid_loss,0.667856273551782 +15,92,avg_train_loss,0.6516686975955963 +15,92,average_valid_loss,0.6708245202898979 +15,96,avg_train_loss,0.5750208497047424 +15,96,average_valid_loss,0.7477074638009071 +16,4,avg_train_loss,0.5183276981115341 +16,4,average_valid_loss,0.7577217444777489 +16,8,avg_train_loss,0.6472078263759613 +16,8,average_valid_loss,0.6820549468199412 +16,12,avg_train_loss,0.6779447197914124 +16,12,average_valid_loss,0.6735506678620974 +16,16,avg_train_loss,0.685187965631485 +16,16,average_valid_loss,0.6707726021607717 +16,20,avg_train_loss,0.6850103586912155 +16,20,average_valid_loss,0.6735146989425024 +16,24,avg_train_loss,0.6554078906774521 +16,24,average_valid_loss,0.6654866710305214 +16,28,avg_train_loss,0.6862213164567947 +16,28,average_valid_loss,0.6702454909682274 +16,32,avg_train_loss,0.6797755509614944 +16,32,average_valid_loss,0.6712867443760236 +16,36,avg_train_loss,0.6850425601005554 +16,36,average_valid_loss,0.6665166392922401 +16,40,avg_train_loss,0.6326764822006226 +16,40,average_valid_loss,0.6690111085772514 +16,44,avg_train_loss,0.6824788600206375 +16,44,average_valid_loss,0.662887970606486 +16,48,avg_train_loss,0.6949313133955002 +16,48,average_valid_loss,0.6641228174169859 +16,52,avg_train_loss,0.6880073994398117 +16,52,average_valid_loss,0.6892866517106692 +16,56,avg_train_loss,0.6816535443067551 +16,56,average_valid_loss,0.6700258627533913 +16,60,avg_train_loss,0.6734307110309601 +16,60,average_valid_loss,0.6746512825290362 +16,64,avg_train_loss,0.6975665092468262 +16,64,average_valid_loss,0.669665073355039 +16,68,avg_train_loss,0.6808061748743057 +16,68,average_valid_loss,0.7108946914474169 +16,72,avg_train_loss,0.6837795972824097 +16,72,average_valid_loss,0.6635947003960609 +16,76,avg_train_loss,0.6784354746341705 +16,76,average_valid_loss,0.6643801753719648 +16,80,avg_train_loss,0.6842817068099976 +16,80,average_valid_loss,0.6658223271369934 +16,84,avg_train_loss,0.7006358206272125 +16,84,average_valid_loss,0.6779131044944128 +16,88,avg_train_loss,0.6953251361846924 +16,88,average_valid_loss,0.6697941496968269 +16,92,avg_train_loss,0.6491118371486664 +16,92,average_valid_loss,0.6729825834433237 +16,96,avg_train_loss,0.5866539925336838 +16,96,average_valid_loss,0.7280682747562727 +17,4,avg_train_loss,0.5092223882675171 +17,4,average_valid_loss,0.7568058768908182 +17,8,avg_train_loss,0.6370884478092194 +17,8,average_valid_loss,0.6822130878766378 +17,12,avg_train_loss,0.6768813878297806 +17,12,average_valid_loss,0.6729821885625521 +17,16,avg_train_loss,0.6878345012664795 +17,16,average_valid_loss,0.6691097592314085 +17,20,avg_train_loss,0.6848126649856567 +17,20,average_valid_loss,0.6672873248656591 +17,24,avg_train_loss,0.6626971811056137 +17,24,average_valid_loss,0.6655565947294235 +17,28,avg_train_loss,0.6807961761951447 +17,28,average_valid_loss,0.67585056523482 +17,32,avg_train_loss,0.6743134707212448 +17,32,average_valid_loss,0.6727924520770708 +17,36,avg_train_loss,0.670731246471405 +17,36,average_valid_loss,0.6626705800493559 +17,40,avg_train_loss,0.6295684725046158 +17,40,average_valid_loss,0.6661520749330521 +17,44,avg_train_loss,0.6746608465909958 +17,44,average_valid_loss,0.6572867135206858 +17,48,avg_train_loss,0.7140950709581375 +17,48,average_valid_loss,0.6652075971166292 +17,52,avg_train_loss,0.6919526755809784 +17,52,average_valid_loss,0.6811424220601717 +17,56,avg_train_loss,0.6712287962436676 +17,56,average_valid_loss,0.6756725236773491 +17,60,avg_train_loss,0.6788829416036606 +17,60,average_valid_loss,0.6663854420185089 +17,64,avg_train_loss,0.7024051547050476 +17,64,average_valid_loss,0.6687072043617567 +17,68,avg_train_loss,0.688066378235817 +17,68,average_valid_loss,0.6967772791783015 +17,72,avg_train_loss,0.6925892978906631 +17,72,average_valid_loss,0.6678594574332237 +17,76,avg_train_loss,0.6800853312015533 +17,76,average_valid_loss,0.6666258399685224 +17,80,avg_train_loss,0.6709132343530655 +17,80,average_valid_loss,0.6680405512452126 +17,84,avg_train_loss,0.6904609948396683 +17,84,average_valid_loss,0.7004822169740995 +17,88,avg_train_loss,0.6864340752363205 +17,88,average_valid_loss,0.664211317896843 +17,92,avg_train_loss,0.6723888963460922 +17,92,average_valid_loss,0.6707966377337774 +17,96,avg_train_loss,0.5882029384374619 +17,96,average_valid_loss,0.7278753866751989 +18,4,avg_train_loss,0.4993241876363754 +18,4,average_valid_loss,0.7636696671446165 +18,8,avg_train_loss,0.6531377136707306 +18,8,average_valid_loss,0.6777105629444122 +18,12,avg_train_loss,0.6766866147518158 +18,12,average_valid_loss,0.6724883144100507 +18,16,avg_train_loss,0.677100270986557 +18,16,average_valid_loss,0.6669824073712031 +18,20,avg_train_loss,0.6827060580253601 +18,20,average_valid_loss,0.6677874053517977 +18,24,avg_train_loss,0.6565988212823868 +18,24,average_valid_loss,0.6672956248124441 +18,28,avg_train_loss,0.6931440532207489 +18,28,average_valid_loss,0.6757699350516001 +18,32,avg_train_loss,0.6648362725973129 +18,32,average_valid_loss,0.6740437845389048 +18,36,avg_train_loss,0.6700883507728577 +18,36,average_valid_loss,0.6583147098620733 +18,40,avg_train_loss,0.661902517080307 +18,40,average_valid_loss,0.6597391540805498 +18,44,avg_train_loss,0.6920823603868484 +18,44,average_valid_loss,0.6632099996010462 +18,48,avg_train_loss,0.710934117436409 +18,48,average_valid_loss,0.6652508676052094 +18,52,avg_train_loss,0.679330587387085 +18,52,average_valid_loss,0.6823472504814466 +18,56,avg_train_loss,0.6614902019500732 +18,56,average_valid_loss,0.6615247776110967 +18,60,avg_train_loss,0.6756769120693207 +18,60,average_valid_loss,0.6702239165703455 +18,64,avg_train_loss,0.6925110667943954 +18,64,average_valid_loss,0.6694436818361282 +18,68,avg_train_loss,0.6682714968919754 +18,68,average_valid_loss,0.6976954365770022 +18,72,avg_train_loss,0.6940854489803314 +18,72,average_valid_loss,0.6599390258391699 +18,76,avg_train_loss,0.6822250485420227 +18,76,average_valid_loss,0.6581282069285711 +18,80,avg_train_loss,0.6871582716703415 +18,80,average_valid_loss,0.6678753644227982 +18,84,avg_train_loss,0.6781442165374756 +18,84,average_valid_loss,0.6795226509372393 +18,88,avg_train_loss,0.6854944378137589 +18,88,average_valid_loss,0.6612179577350616 +18,92,avg_train_loss,0.6325002610683441 +18,92,average_valid_loss,0.6604203283786774 +18,96,avg_train_loss,0.5438236743211746 +18,96,average_valid_loss,0.7951959744095802 +19,4,avg_train_loss,0.5319216847419739 +19,4,average_valid_loss,0.7392773206035296 +19,8,avg_train_loss,0.6458642333745956 +19,8,average_valid_loss,0.6698619772990545 +19,12,avg_train_loss,0.6663940846920013 +19,12,average_valid_loss,0.665406291683515 +19,16,avg_train_loss,0.6871741861104965 +19,16,average_valid_loss,0.6615699231624603 +19,20,avg_train_loss,0.6763308644294739 +19,20,average_valid_loss,0.6657046948870023 +19,24,avg_train_loss,0.6594454050064087 +19,24,average_valid_loss,0.660836694141229 +19,28,avg_train_loss,0.6825186014175415 +19,28,average_valid_loss,0.662944974998633 +19,32,avg_train_loss,0.6684815436601639 +19,32,average_valid_loss,0.6550267413258553 +19,36,avg_train_loss,0.6625236570835114 +19,36,average_valid_loss,0.651306502521038 +19,40,avg_train_loss,0.6349002718925476 +19,40,average_valid_loss,0.6537862196564674 +19,44,avg_train_loss,0.687857374548912 +19,44,average_valid_loss,0.6587710529565811 +19,48,avg_train_loss,0.6830068826675415 +19,48,average_valid_loss,0.6566637630263964 +19,52,avg_train_loss,0.6756454855203629 +19,52,average_valid_loss,0.673285556336244 +19,56,avg_train_loss,0.6692714989185333 +19,56,average_valid_loss,0.6654625634352366 +19,60,avg_train_loss,0.6872029155492783 +19,60,average_valid_loss,0.654798778394858 +19,64,avg_train_loss,0.6831528693437576 +19,64,average_valid_loss,0.6666117956240972 +19,68,avg_train_loss,0.6761450469493866 +19,68,average_valid_loss,0.6875940064589182 +19,72,avg_train_loss,0.6711909025907516 +19,72,average_valid_loss,0.6543336560328802 +19,76,avg_train_loss,0.6861682087182999 +19,76,average_valid_loss,0.6551840156316757 +19,80,avg_train_loss,0.6823762804269791 +19,80,average_valid_loss,0.6622807309031487 +19,84,avg_train_loss,0.6682124435901642 +19,84,average_valid_loss,0.675057386358579 +19,88,avg_train_loss,0.6692050099372864 +19,88,average_valid_loss,0.6557515462239584 +19,92,avg_train_loss,0.6403584182262421 +19,92,average_valid_loss,0.6543424973885218 +19,96,avg_train_loss,0.5794831439852715 +19,96,average_valid_loss,0.7363407959540685 +20,4,avg_train_loss,0.5115397423505783 +20,4,average_valid_loss,0.7518377626935641 +20,8,avg_train_loss,0.6405661404132843 +20,8,average_valid_loss,0.6700947706898054 +20,12,avg_train_loss,0.6702582389116287 +20,12,average_valid_loss,0.664151189227899 +20,16,avg_train_loss,0.6924922466278076 +20,16,average_valid_loss,0.6647224873304367 +20,20,avg_train_loss,0.6606941223144531 +20,20,average_valid_loss,0.6611894915501276 +20,24,avg_train_loss,0.6609189808368683 +20,24,average_valid_loss,0.6563023204604784 +20,28,avg_train_loss,0.6837473213672638 +20,28,average_valid_loss,0.6700323422749838 +20,32,avg_train_loss,0.6445568948984146 +20,32,average_valid_loss,0.6631559357047081 +20,36,avg_train_loss,0.6493311524391174 +20,36,average_valid_loss,0.6451436554392179 +20,40,avg_train_loss,0.652996838092804 +20,40,average_valid_loss,0.6496002078056335 +20,44,avg_train_loss,0.6785660237073898 +20,44,average_valid_loss,0.6549332117040952 +20,48,avg_train_loss,0.7052848637104034 +20,48,average_valid_loss,0.6564752335349718 +20,52,avg_train_loss,0.6527944207191467 +20,52,average_valid_loss,0.666240374247233 +20,56,avg_train_loss,0.6429008394479752 +20,56,average_valid_loss,0.6902404973904291 +20,60,avg_train_loss,0.7067168653011322 +20,60,average_valid_loss,0.653696745634079 +20,64,avg_train_loss,0.6752861440181732 +20,64,average_valid_loss,0.6626725693543752 +20,68,avg_train_loss,0.6659803539514542 +20,68,average_valid_loss,0.7145467276374499 +20,72,avg_train_loss,0.6866470873355865 +20,72,average_valid_loss,0.6538533568382263 +20,76,avg_train_loss,0.6772201955318451 +20,76,average_valid_loss,0.6499591295917829 +20,80,avg_train_loss,0.6752555668354034 +20,80,average_valid_loss,0.664854146540165 +20,84,avg_train_loss,0.649316817522049 +20,84,average_valid_loss,0.6631953070561091 +20,88,avg_train_loss,0.6928933560848236 +20,88,average_valid_loss,0.6633235389987627 +20,92,avg_train_loss,0.6449453085660934 +20,92,average_valid_loss,0.6562504743536314 +20,96,avg_train_loss,0.5819380283355713 +20,96,average_valid_loss,0.7151729414860407 diff --git a/.flor/run.ipy b/.flor/run.ipy new file mode 100644 index 0000000..01e4ea8 --- /dev/null +++ b/.flor/run.ipy @@ -0,0 +1,2 @@ +# coding: utf-8 +!python train_rnn.py --replay_flor 17/20 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index b6e4761..f47e367 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ __pycache__/ *.py[cod] *$py.class - +.venv/ # C extensions *.so @@ -99,31 +99,8 @@ celerybeat-schedule celerybeat.pid # SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ +.mypy_cache/** + +dump/** +.DS_Store +peak_pivot.csv diff --git a/.replay.json b/.replay.json deleted file mode 100644 index ac919ac..0000000 --- a/.replay.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "NAME": "kaggle-nlp-disasters", - "MEMO": "/home/codespace/.flor/kaggle-nlp-disasters/2022-01-22T22:35:34.json", - "KVS": { - "clf_seed": 1328 - } -} \ No newline at end of file diff --git a/.train_rnn_tfm.py b/.train_rnn_tfm.py new file mode 100755 index 0000000..fb47436 --- /dev/null +++ b/.train_rnn_tfm.py @@ -0,0 +1,268 @@ +# type: ignore + +import flor +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +import pandas as pd +import torch +from torchtext.legacy.data import Field, TabularDataset, BucketIterator +import torch.nn as nn +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence +import torch.optim as optim +from sklearn.metrics import accuracy_score, classification_report, confusion_matrix +import seaborn as sns +import flor +from multiprocessing import set_start_method +from utils import CLR_Scheduler +try: + set_start_method('spawn') +except RuntimeError: + pass +device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') +flor.namespace_stack.test_force(device, 'device') +device +label_field = Field(sequential=False, use_vocab=False, batch_first=True, + dtype=torch.float) +flor.namespace_stack.test_force(label_field, 'label_field') +text_field = Field(tokenize='spacy', lower=True, include_lengths=True, + batch_first=True) +flor.namespace_stack.test_force(text_field, 'text_field') +fields = [('words', text_field), ('target', label_field)] +flor.namespace_stack.test_force(fields, 'fields') +fields_test = [('words', text_field)] +flor.namespace_stack.test_force(fields_test, 'fields_test') +train, valid = TabularDataset.splits(path='data', train='train_rnn.csv', + validation='valid_rnn.csv', format='CSV', fields=fields, skip_header=True) +flor.namespace_stack.test_force(train, 'train') +flor.namespace_stack.test_force(valid, 'valid') +test = TabularDataset(path='data/test_rnn.csv', format='CSV', fields= + fields_test, skip_header=True) +flor.namespace_stack.test_force(test, 'test') +train_iter = BucketIterator(train, batch_size=200, sort_key=lambda x: len(x + .words), device=device, sort=True, sort_within_batch=True) +flor.namespace_stack.test_force(train_iter, 'train_iter') +valid_iter = BucketIterator(valid, batch_size=200, sort_key=lambda x: len(x + .words), device=device, sort=True, sort_within_batch=True) +flor.namespace_stack.test_force(valid_iter, 'valid_iter') +test_iter = BucketIterator(test, batch_size=200, sort_key=lambda x: len(x. + words), device=device, sort=True, sort_within_batch=True) +flor.namespace_stack.test_force(test_iter, 'test_iter') +text_field.build_vocab(train, min_freq=5) + + +class LSTM(nn.Module): + + def __init__(self, dimension=128): + try: + flor.namespace_stack.new() + super(LSTM, self).__init__() + self.embedding = nn.Embedding(len(text_field.vocab), dimension) + flor.namespace_stack.test_force(self.embedding, 'self.embedding') + self.lstm = nn.LSTM(input_size=dimension, hidden_size=dimension, + num_layers=1, batch_first=True, bidirectional=True) + flor.namespace_stack.test_force(self.lstm, 'self.lstm') + self.drop = nn.Dropout(p=0.85) + flor.namespace_stack.test_force(self.drop, 'self.drop') + self.dimension = dimension + flor.namespace_stack.test_force(self.dimension, 'self.dimension') + self.fc = nn.Linear(2 * dimension, 1) + flor.namespace_stack.test_force(self.fc, 'self.fc') + self.relu = nn.ReLU() + flor.namespace_stack.test_force(self.relu, 'self.relu') + finally: + flor.namespace_stack.pop() + + def forward(self, text, text_len): + try: + flor.namespace_stack.new() + text_emb = self.relu(self.embedding(text)) + flor.namespace_stack.test_force(text_emb, 'text_emb') + packed_input = pack_padded_sequence(text_emb, text_len, + batch_first=True, enforce_sorted=False) + flor.namespace_stack.test_force(packed_input, 'packed_input') + packed_output, _ = self.lstm(packed_input) + flor.namespace_stack.test_force(packed_output, 'packed_output') + flor.namespace_stack.test_force(_, '_') + output, _ = pad_packed_sequence(packed_output, batch_first=True) + flor.namespace_stack.test_force(output, 'output') + flor.namespace_stack.test_force(_, '_') + out_forward = output[(range(len(output))), (text_len - 1), : + self.dimension] + flor.namespace_stack.test_force(out_forward, 'out_forward') + out_reverse = output[:, (0), self.dimension:] + flor.namespace_stack.test_force(out_reverse, 'out_reverse') + out_reduced = torch.cat((out_forward, out_reverse), 1) + flor.namespace_stack.test_force(out_reduced, 'out_reduced') + text_fea = out_reduced + flor.namespace_stack.test_force(text_fea, 'text_fea') + text_fea = self.fc(self.drop(text_fea)) + flor.namespace_stack.test_force(text_fea, 'text_fea') + text_fea = torch.squeeze(text_fea, 1) + flor.namespace_stack.test_force(text_fea, 'text_fea') + text_out = torch.sigmoid(text_fea) + flor.namespace_stack.test_force(text_out, 'text_out') + return text_out + finally: + flor.namespace_stack.pop() + + +def train(model, optimizer, criterion=nn.BCELoss(), train_loader=train_iter, + valid_loader=valid_iter, test_loader=test_iter, num_epochs=5, + eval_every=len(train_iter) // 2, file_path='training_process', + best_valid_loss=float('Inf')): + try: + flor.namespace_stack.new() + running_loss = 0.0 + flor.namespace_stack.test_force(running_loss, 'running_loss') + valid_running_loss = 0.0 + flor.namespace_stack.test_force(valid_running_loss, + 'valid_running_loss') + global_step = 0 + flor.namespace_stack.test_force(global_step, 'global_step') + train_loss_list = [] + flor.namespace_stack.test_force(train_loss_list, 'train_loss_list') + valid_loss_list = [] + flor.namespace_stack.test_force(valid_loss_list, 'valid_loss_list') + global_steps_list = [] + flor.namespace_stack.test_force(global_steps_list, 'global_steps_list') + best_loss = float('inf') + flor.namespace_stack.test_force(best_loss, 'best_loss') + model.train() + flor.skip_stack.new(2) + if flor.skip_stack.peek().should_execute(not flor.SKIP): + for epoch in flor.it(range(num_epochs)): + flor.log('learning_rate', str(optimizer.param_groups[0]['lr'])) + flor.skip_stack.new(1) + if flor.skip_stack.peek().should_execute(not flor.SKIP): + for ((words, words_len), labels), _ in train_loader: + labels = labels.to(device) + flor.namespace_stack.test_force(labels, 'labels') + words = words.to(device) + flor.namespace_stack.test_force(words, 'words') + words_len = words_len.detach().cpu() + flor.namespace_stack.test_force(words_len, + 'words_len') + output = model(words, words_len) + flor.namespace_stack.test_force(output, 'output') + loss = criterion(output, labels) + flor.namespace_stack.test_force(loss, 'loss') + optimizer.zero_grad() + loss.backward() + optimizer.step() + running_loss += loss.item() + global_step += 1 + if global_step % eval_every == 0: + model.eval() + with torch.no_grad(): + flor.skip_stack.new(0) + if flor.skip_stack.peek().should_execute( + not flor.SKIP): + for ((words, words_len), labels + ), _ in valid_loader: + labels = labels.to(device) + flor.namespace_stack.test_force(labels, + 'labels') + words = words.to(device) + flor.namespace_stack.test_force(words, + 'words') + words_len = words_len.detach().cpu() + flor.namespace_stack.test_force(words_len, + 'words_len') + output = model(words, words_len) + flor.namespace_stack.test_force(output, + 'output') + loss = criterion(output, labels) + flor.namespace_stack.test_force(loss, + 'loss') + valid_running_loss += float(loss.item()) + (valid_running_loss, _, _, words_len, + output, loss) = (flor.skip_stack. + pop().proc_side_effects( + valid_running_loss, labels, words, + words_len, output, loss)) + average_train_loss = running_loss / eval_every + flor.namespace_stack.test_force( + average_train_loss, 'average_train_loss') + average_valid_loss = valid_running_loss / len( + valid_loader) + flor.namespace_stack.test_force( + average_valid_loss, 'average_valid_loss') + if average_valid_loss < best_loss: + best_loss = average_valid_loss + flor.namespace_stack.test_force(best_loss, + 'best_loss') + torch.save(model.state_dict(), + 'best-model.pt') + train_loss_list.append(average_train_loss) + valid_loss_list.append(average_valid_loss) + global_steps_list.append(global_step) + running_loss = 0.0 + flor.namespace_stack.test_force(running_loss, + 'running_loss') + valid_running_loss = 0.0 + flor.namespace_stack.test_force( + valid_running_loss, 'valid_running_loss') + model.train() + print( + 'Epoch [{}/{}], LR: {:.3f}, Step [{}/{}], Train Loss: {:.4f}, Valid Loss: {:.4f}' + .format(epoch + 1, num_epochs, + optimizer.param_groups[0]['lr'], + global_step, num_epochs * len( + train_loader), average_train_loss, + average_valid_loss)) + flor.log('avg_train_loss', average_train_loss) + flor.log('average_valid_loss', + average_valid_loss) + clr_scheduler.step() + (_, _, running_loss, valid_running_loss, global_step, _, + _, _, best_loss, _, _, _) = (flor.skip_stack.pop(). + proc_side_effects(model, optimizer, running_loss, + valid_running_loss, global_step, train_loss_list, + valid_loss_list, global_steps_list, best_loss, + torch, flor, clr_scheduler)) + (_, _, running_loss, valid_running_loss, global_step, _, _, _, + best_loss, _, _, _) = (flor.skip_stack.pop().proc_side_effects( + model, optimizer, running_loss, valid_running_loss, global_step, + train_loss_list, valid_loss_list, global_steps_list, best_loss, + flor, torch, clr_scheduler)) + y_pred = [] + flor.namespace_stack.test_force(y_pred, 'y_pred') + model.eval() + with torch.no_grad(): + flor.skip_stack.new(3) + if flor.skip_stack.peek().should_execute(not flor.SKIP): + for (words, words_len), _ in test_loader: + words = words.to(device) + flor.namespace_stack.test_force(words, 'words') + words_len = words_len.detach().cpu() + flor.namespace_stack.test_force(words_len, 'words_len') + output = model(words, words_len) + flor.namespace_stack.test_force(output, 'output') + output = (output > 0.5).int() + flor.namespace_stack.test_force(output, 'output') + y_pred.extend(output.tolist()) + _, words_len, output, _ = flor.skip_stack.pop().proc_side_effects( + words, words_len, output, y_pred) + print('Finished Training!') + return y_pred + finally: + flor.namespace_stack.pop() + + +EPOCHS = 80 +flor.namespace_stack.test_force(EPOCHS, 'EPOCHS') +MIN_LR = 0.0001 +flor.namespace_stack.test_force(MIN_LR, 'MIN_LR') +model = LSTM(8).to(device) +flor.namespace_stack.test_force(model, 'model') +optimizer = optim.SGD(model.parameters(), lr=MIN_LR) +flor.namespace_stack.test_force(optimizer, 'optimizer') +flor.log('optimizer', str(type(optimizer))) +clr_scheduler = CLR_Scheduler(optimizer, net_steps=len(train_iter) * EPOCHS, + min_lr=MIN_LR, max_lr=4.0, tail_frac=0.0) +flor.namespace_stack.test_force(clr_scheduler, 'clr_scheduler') +pred = train(model=model, optimizer=optimizer, num_epochs=EPOCHS) +flor.namespace_stack.test_force(pred, 'pred') +if not flor.SKIP: + flor.flush() diff --git a/.venv/bin/Activate.ps1 b/.venv/bin/Activate.ps1 deleted file mode 100644 index 2fb3852..0000000 --- a/.venv/bin/Activate.ps1 +++ /dev/null @@ -1,241 +0,0 @@ -<# -.Synopsis -Activate a Python virtual environment for the current PowerShell session. - -.Description -Pushes the python executable for a virtual environment to the front of the -$Env:PATH environment variable and sets the prompt to signify that you are -in a Python virtual environment. Makes use of the command line switches as -well as the `pyvenv.cfg` file values present in the virtual environment. - -.Parameter VenvDir -Path to the directory that contains the virtual environment to activate. The -default value for this is the parent of the directory that the Activate.ps1 -script is located within. - -.Parameter Prompt -The prompt prefix to display when this virtual environment is activated. By -default, this prompt is the name of the virtual environment folder (VenvDir) -surrounded by parentheses and followed by a single space (ie. '(.venv) '). - -.Example -Activate.ps1 -Activates the Python virtual environment that contains the Activate.ps1 script. - -.Example -Activate.ps1 -Verbose -Activates the Python virtual environment that contains the Activate.ps1 script, -and shows extra information about the activation as it executes. - -.Example -Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv -Activates the Python virtual environment located in the specified location. - -.Example -Activate.ps1 -Prompt "MyPython" -Activates the Python virtual environment that contains the Activate.ps1 script, -and prefixes the current prompt with the specified string (surrounded in -parentheses) while the virtual environment is active. - -.Notes -On Windows, it may be required to enable this Activate.ps1 script by setting the -execution policy for the user. You can do this by issuing the following PowerShell -command: - -PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - -For more information on Execution Policies: -https://go.microsoft.com/fwlink/?LinkID=135170 - -#> -Param( - [Parameter(Mandatory = $false)] - [String] - $VenvDir, - [Parameter(Mandatory = $false)] - [String] - $Prompt -) - -<# Function declarations --------------------------------------------------- #> - -<# -.Synopsis -Remove all shell session elements added by the Activate script, including the -addition of the virtual environment's Python executable from the beginning of -the PATH variable. - -.Parameter NonDestructive -If present, do not remove this function from the global namespace for the -session. - -#> -function global:deactivate ([switch]$NonDestructive) { - # Revert to original values - - # The prior prompt: - if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { - Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt - Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT - } - - # The prior PYTHONHOME: - if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { - Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME - Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME - } - - # The prior PATH: - if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { - Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH - Remove-Item -Path Env:_OLD_VIRTUAL_PATH - } - - # Just remove the VIRTUAL_ENV altogether: - if (Test-Path -Path Env:VIRTUAL_ENV) { - Remove-Item -Path env:VIRTUAL_ENV - } - - # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: - if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { - Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force - } - - # Leave deactivate function in the global namespace if requested: - if (-not $NonDestructive) { - Remove-Item -Path function:deactivate - } -} - -<# -.Description -Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the -given folder, and returns them in a map. - -For each line in the pyvenv.cfg file, if that line can be parsed into exactly -two strings separated by `=` (with any amount of whitespace surrounding the =) -then it is considered a `key = value` line. The left hand string is the key, -the right hand is the value. - -If the value starts with a `'` or a `"` then the first and last character is -stripped from the value before being captured. - -.Parameter ConfigDir -Path to the directory that contains the `pyvenv.cfg` file. -#> -function Get-PyVenvConfig( - [String] - $ConfigDir -) { - Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" - - # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). - $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue - - # An empty map will be returned if no config file is found. - $pyvenvConfig = @{ } - - if ($pyvenvConfigPath) { - - Write-Verbose "File exists, parse `key = value` lines" - $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath - - $pyvenvConfigContent | ForEach-Object { - $keyval = $PSItem -split "\s*=\s*", 2 - if ($keyval[0] -and $keyval[1]) { - $val = $keyval[1] - - # Remove extraneous quotations around a string value. - if ("'""".Contains($val.Substring(0, 1))) { - $val = $val.Substring(1, $val.Length - 2) - } - - $pyvenvConfig[$keyval[0]] = $val - Write-Verbose "Adding Key: '$($keyval[0])'='$val'" - } - } - } - return $pyvenvConfig -} - - -<# Begin Activate script --------------------------------------------------- #> - -# Determine the containing directory of this script -$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$VenvExecDir = Get-Item -Path $VenvExecPath - -Write-Verbose "Activation script is located in path: '$VenvExecPath'" -Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" -Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" - -# Set values required in priority: CmdLine, ConfigFile, Default -# First, get the location of the virtual environment, it might not be -# VenvExecDir if specified on the command line. -if ($VenvDir) { - Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" -} -else { - Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." - $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") - Write-Verbose "VenvDir=$VenvDir" -} - -# Next, read the `pyvenv.cfg` file to determine any required value such -# as `prompt`. -$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir - -# Next, set the prompt from the command line, or the config file, or -# just use the name of the virtual environment folder. -if ($Prompt) { - Write-Verbose "Prompt specified as argument, using '$Prompt'" -} -else { - Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" - if ($pyvenvCfg -and $pyvenvCfg['prompt']) { - Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" - $Prompt = $pyvenvCfg['prompt']; - } - else { - Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virutal environment)" - Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" - $Prompt = Split-Path -Path $venvDir -Leaf - } -} - -Write-Verbose "Prompt = '$Prompt'" -Write-Verbose "VenvDir='$VenvDir'" - -# Deactivate any currently active virtual environment, but leave the -# deactivate function in place. -deactivate -nondestructive - -# Now set the environment variable VIRTUAL_ENV, used by many tools to determine -# that there is an activated venv. -$env:VIRTUAL_ENV = $VenvDir - -if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { - - Write-Verbose "Setting prompt to '$Prompt'" - - # Set the prompt to include the env name - # Make sure _OLD_VIRTUAL_PROMPT is global - function global:_OLD_VIRTUAL_PROMPT { "" } - Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT - New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt - - function global:prompt { - Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " - _OLD_VIRTUAL_PROMPT - } -} - -# Clear PYTHONHOME -if (Test-Path -Path Env:PYTHONHOME) { - Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME - Remove-Item -Path Env:PYTHONHOME -} - -# Add the venv to the PATH -Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH -$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/.venv/bin/activate b/.venv/bin/activate deleted file mode 100644 index c5ae37f..0000000 --- a/.venv/bin/activate +++ /dev/null @@ -1,76 +0,0 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - # reset old environment variables - if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then - PATH="${_OLD_VIRTUAL_PATH:-}" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then - PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r - fi - - if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then - PS1="${_OLD_VIRTUAL_PS1:-}" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - if [ ! "${1:-}" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelevant variables -deactivate nondestructive - -VIRTUAL_ENV="/workspaces/kaggle-nlp-disasters/.venv" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/bin:$PATH" -export PATH - -# unset PYTHONHOME if set -# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) -# could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "${PYTHONHOME:-}" ] ; then - _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" - unset PYTHONHOME -fi - -if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then - _OLD_VIRTUAL_PS1="${PS1:-}" - if [ "x(.venv) " != x ] ; then - PS1="(.venv) ${PS1:-}" - else - if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then - # special case for Aspen magic directories - # see https://aspen.io/ - PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" - else - PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" - fi - fi - export PS1 -fi - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r -fi diff --git a/.venv/bin/activate.csh b/.venv/bin/activate.csh deleted file mode 100644 index 83f67bc..0000000 --- a/.venv/bin/activate.csh +++ /dev/null @@ -1,37 +0,0 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi . -# Ported to Python 3.3 venv by Andrew Svetlov - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' - -# Unset irrelevant variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "/workspaces/kaggle-nlp-disasters/.venv" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/bin:$PATH" - - -set _OLD_VIRTUAL_PROMPT="$prompt" - -if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - if (".venv" != "") then - set env_name = ".venv" - else - if (`basename "VIRTUAL_ENV"` == "__") then - # special case for Aspen magic directories - # see https://aspen.io/ - set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` - else - set env_name = `basename "$VIRTUAL_ENV"` - endif - endif - set prompt = "[$env_name] $prompt" - unset env_name -endif - -alias pydoc python -m pydoc - -rehash diff --git a/.venv/bin/activate.fish b/.venv/bin/activate.fish deleted file mode 100644 index b9ad18a..0000000 --- a/.venv/bin/activate.fish +++ /dev/null @@ -1,75 +0,0 @@ -# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) -# you cannot run it directly - -function deactivate -d "Exit virtualenv and return to normal shell environment" - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt - set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt - end - - set -e VIRTUAL_ENV - if test "$argv[1]" != "nondestructive" - # Self destruct! - functions -e deactivate - end -end - -# unset irrelevant variables -deactivate nondestructive - -set -gx VIRTUAL_ENV "/workspaces/kaggle-nlp-disasters/.venv" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/bin" $PATH - -# unset PYTHONHOME if set -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # fish uses a function instead of an env var to generate the prompt. - - # save the current fish_prompt function as the function _old_fish_prompt - functions -c fish_prompt _old_fish_prompt - - # with the original prompt function renamed, we can override with our own. - function fish_prompt - # Save the return status of the last command - set -l old_status $status - - # Prompt override? - if test -n "(.venv) " - printf "%s%s" "(.venv) " (set_color normal) - else - # ...Otherwise, prepend env - set -l _checkbase (basename "$VIRTUAL_ENV") - if test $_checkbase = "__" - # special case for Aspen magic directories - # see https://aspen.io/ - printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) - else - printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) - end - end - - # Restore the return status of the previous command. - echo "exit $old_status" | . - _old_fish_prompt - end - - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" -end diff --git a/.venv/bin/pip b/.venv/bin/pip deleted file mode 100755 index 4e67822..0000000 --- a/.venv/bin/pip +++ /dev/null @@ -1,8 +0,0 @@ -#!/workspaces/kaggle-nlp-disasters/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.venv/bin/pip3 b/.venv/bin/pip3 deleted file mode 100755 index 4e67822..0000000 --- a/.venv/bin/pip3 +++ /dev/null @@ -1,8 +0,0 @@ -#!/workspaces/kaggle-nlp-disasters/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.venv/bin/pip3.8 b/.venv/bin/pip3.8 deleted file mode 100755 index 4e67822..0000000 --- a/.venv/bin/pip3.8 +++ /dev/null @@ -1,8 +0,0 @@ -#!/workspaces/kaggle-nlp-disasters/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.venv/bin/python b/.venv/bin/python deleted file mode 120000 index 7ae5fe3..0000000 --- a/.venv/bin/python +++ /dev/null @@ -1 +0,0 @@ -/home/codespace/.python/current/bin/python \ No newline at end of file diff --git a/.venv/bin/python3 b/.venv/bin/python3 deleted file mode 120000 index d8654aa..0000000 --- a/.venv/bin/python3 +++ /dev/null @@ -1 +0,0 @@ -python \ No newline at end of file diff --git a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/AUTHORS b/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/AUTHORS deleted file mode 100644 index 55d6818..0000000 --- a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/AUTHORS +++ /dev/null @@ -1,48 +0,0 @@ -GitPython was originally written by Michael Trier. -GitPython 0.2 was partially (re)written by Sebastian Thiel, based on 0.1.6 and git-dulwich. - -Contributors are: - --Michael Trier --Alan Briolat --Florian Apolloner --David Aguilar --Jelmer Vernooij --Steve Frécinaux --Kai Lautaportti --Paul Sowden --Sebastian Thiel --Jonathan Chu --Vincent Driessen --Phil Elson --Bernard `Guyzmo` Pratz --Timothy B. Hartman --Konstantin Popov --Peter Jones --Anson Mansfield --Ken Odegard --Alexis Horgix Chotard --Piotr Babij --Mikuláš Poul --Charles Bouchard-Légaré --Yaroslav Halchenko --Tim Swast --William Luc Ritchie --David Host --A. Jesse Jiryu Davis --Steven Whitman --Stefan Stancu --César Izurieta --Arthur Milchior --Anil Khatri --JJ Graham --Ben Thayer --Dries Kennes --Pratik Anurag --Harmon --Liam Beguin --Ram Rachum --Alba Mendez --Robert Westman --Hugo van Kemenade -Portions derived from other open source works and are clearly marked. diff --git a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/INSTALLER b/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/INSTALLER deleted file mode 100644 index a1b589e..0000000 --- a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/LICENSE b/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/LICENSE deleted file mode 100644 index 5a9a6f8..0000000 --- a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/LICENSE +++ /dev/null @@ -1,30 +0,0 @@ -Copyright (C) 2008, 2009 Michael Trier and contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - -* Neither the name of the GitPython project nor the names of -its contributors may be used to endorse or promote products derived -from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/METADATA b/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/METADATA deleted file mode 100644 index 5e20cd5..0000000 --- a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/METADATA +++ /dev/null @@ -1,32 +0,0 @@ -Metadata-Version: 2.1 -Name: GitPython -Version: 3.1.26 -Summary: GitPython is a python library used to interact with Git repositories -Home-page: https://github.com/gitpython-developers/GitPython -Author: Sebastian Thiel, Michael Trier -Author-email: byronimo@gmail.com, mtrier@gmail.com -License: BSD -Platform: UNKNOWN -Classifier: Development Status :: 5 - Production/Stable -Classifier: Environment :: Console -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: BSD License -Classifier: Operating System :: OS Independent -Classifier: Operating System :: POSIX -Classifier: Operating System :: Microsoft :: Windows -Classifier: Operating System :: MacOS :: MacOS X -Classifier: Typing :: Typed -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Requires-Python: >=3.7 -Description-Content-Type: text/markdown -Requires-Dist: gitdb (<5,>=4.0.1) -Requires-Dist: typing-extensions (>=3.7.4.3) ; python_version < "3.8" - -GitPython is a python library used to interact with Git repositories - - diff --git a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/RECORD b/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/RECORD deleted file mode 100644 index 3ed7ee5..0000000 --- a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/RECORD +++ /dev/null @@ -1,82 +0,0 @@ -GitPython-3.1.26.dist-info/AUTHORS,sha256=vjnd09wZL3p1v8gB5lsk4nj-2nDyHcZEzY_MKrZQyco,1936 -GitPython-3.1.26.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -GitPython-3.1.26.dist-info/LICENSE,sha256=_WV__CzvY9JceMq3gI1BTdA6KC5jiTSR_RHDL5i-Z_s,1521 -GitPython-3.1.26.dist-info/METADATA,sha256=WBgoy2zwHyQWGMKqWoJb6u3BhdMDvyiT8yqhnWsmat0,1246 -GitPython-3.1.26.dist-info/RECORD,, -GitPython-3.1.26.dist-info/WHEEL,sha256=U88EhGIw8Sj2_phqajeu_EAi3RAo8-C6zV3REsWbWbs,92 -GitPython-3.1.26.dist-info/top_level.txt,sha256=0hzDuIp8obv624V3GmbqsagBWkk8ohtGU-Bc1PmTT0o,4 -git/__init__.py,sha256=xbVhsCMTO6l8RiiKws-4UW1kNdjm-VvkNcVszLRvWuA,2500 -git/__pycache__/__init__.cpython-38.pyc,, -git/__pycache__/cmd.cpython-38.pyc,, -git/__pycache__/compat.cpython-38.pyc,, -git/__pycache__/config.cpython-38.pyc,, -git/__pycache__/db.cpython-38.pyc,, -git/__pycache__/diff.cpython-38.pyc,, -git/__pycache__/exc.cpython-38.pyc,, -git/__pycache__/remote.cpython-38.pyc,, -git/__pycache__/types.cpython-38.pyc,, -git/__pycache__/util.cpython-38.pyc,, -git/cmd.py,sha256=N6N9aEmeOz8dpOlUC2Zo9pbB_aFHxVjWtvYZ-MJvi48,51873 -git/compat.py,sha256=A__j0NDexK9vm-IP59CveY7V_Epef9Km4wH94nykvGs,2244 -git/config.py,sha256=ABfTzzFy82NkwTKHreMbJF78JC43dAR385LbrUhnc84,34506 -git/db.py,sha256=Ji8Zrdq5Gvo3Hm54gBB7DXFYJE0SgPmoHyAp_5RI3s0,2255 -git/diff.py,sha256=nzQ2ZtSFrvcpgTvKBKlheLLZGdLgmW3xbwjaZQtaJBQ,22587 -git/exc.py,sha256=rtiF2l2ZONIih_yF3DZ_WuDXAocdq0inhjQzGJq5T7o,6079 -git/index/__init__.py,sha256=43ovvVNocVRNiQd4fLqvUMuGGmwhBQ9SsiQ46vkvk1E,89 -git/index/__pycache__/__init__.cpython-38.pyc,, -git/index/__pycache__/base.cpython-38.pyc,, -git/index/__pycache__/fun.cpython-38.pyc,, -git/index/__pycache__/typ.cpython-38.pyc,, -git/index/__pycache__/util.cpython-38.pyc,, -git/index/base.py,sha256=DqbeEgQoRuL2RR_ZftKZWUvfw7cZU7YwFfDp7edUs9Y,56987 -git/index/fun.py,sha256=m8ka-FAwmgJ-sJ0EMKC266W2DqxVruN94ztIo7hm61s,15977 -git/index/typ.py,sha256=8-yL3QhdHXkVaDHfUuk4Kmks1Comrq547Kg48m6H2gA,5516 -git/index/util.py,sha256=t3llCo90s1L_OgPYIqah5AuVU6043XKTaQrmJtUeYjU,3454 -git/objects/__init__.py,sha256=1uMoWicK_mgiQIaikCMsX7uiRWc9US4XUXCouSmH4Dk,703 -git/objects/__pycache__/__init__.cpython-38.pyc,, -git/objects/__pycache__/base.cpython-38.pyc,, -git/objects/__pycache__/blob.cpython-38.pyc,, -git/objects/__pycache__/commit.cpython-38.pyc,, -git/objects/__pycache__/fun.cpython-38.pyc,, -git/objects/__pycache__/tag.cpython-38.pyc,, -git/objects/__pycache__/tree.cpython-38.pyc,, -git/objects/__pycache__/util.cpython-38.pyc,, -git/objects/base.py,sha256=qLgh-OStkOke3yKgwq2ZNFTC45Qsl1UTSI1-fdjR8-w,7759 -git/objects/blob.py,sha256=nXCRt885vuNjI6VRw_fXOZSgQfD9PjXPg3XZIRZkIfM,987 -git/objects/commit.py,sha256=-dXHQvop5HXIqQjuMYIFMNLL2AY9y0a7Gw3ayG8OAHY,25869 -git/objects/fun.py,sha256=vSmm8p4_6ZMOh3Vtwbi65gP1vIFczXF2hXtgHclP-EY,8542 -git/objects/submodule/__init__.py,sha256=OsMeiex7cG6ev2f35IaJ5csH-eXchSoNKCt4HXUG5Ws,93 -git/objects/submodule/__pycache__/__init__.cpython-38.pyc,, -git/objects/submodule/__pycache__/base.cpython-38.pyc,, -git/objects/submodule/__pycache__/root.cpython-38.pyc,, -git/objects/submodule/__pycache__/util.cpython-38.pyc,, -git/objects/submodule/base.py,sha256=0Ndm8ftvfBJB1tcUflX_cRU7PAeNJzsYMQeeLv5ZSE8,58693 -git/objects/submodule/root.py,sha256=cy7wRBLJwqNXGm6bK0tgEucYX7lx-KNQS-WTTlEXvig,18288 -git/objects/submodule/util.py,sha256=iX1EYGDhVrr1PG8729zQPm2GL47FkE9MPqPYC8C_h-o,3358 -git/objects/tag.py,sha256=mqlDG5UyScqHEnwDXRBPwbMcPrTZnhMiezpDM7DkEss,3764 -git/objects/tree.py,sha256=ly7fgePvItjB-B69AvdlFXrekH8MixOMauO31sCpg7E,14292 -git/objects/util.py,sha256=gx1jzp1oiqdlADrWu6oorIF3FwXJZtaprzxPlyY9u8I,22466 -git/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -git/refs/__init__.py,sha256=PMF97jMUcivbCCEJnl2zTs-YtECNFp8rL8GHK8AitXU,203 -git/refs/__pycache__/__init__.cpython-38.pyc,, -git/refs/__pycache__/head.cpython-38.pyc,, -git/refs/__pycache__/log.cpython-38.pyc,, -git/refs/__pycache__/reference.cpython-38.pyc,, -git/refs/__pycache__/remote.cpython-38.pyc,, -git/refs/__pycache__/symbolic.cpython-38.pyc,, -git/refs/__pycache__/tag.cpython-38.pyc,, -git/refs/head.py,sha256=UbIuWCb9WI87DZY6puX92YaHLDnh-D6DkONwDjyxcvM,9626 -git/refs/log.py,sha256=8ZTiE7KV4SVAlx14xR7xVG1uh_jFgsSNZ_6QSMh4vpA,12061 -git/refs/reference.py,sha256=28aB_lnYLKIXx2wvy_y87P0EDIuNiCsGLgBmV-29qP0,5405 -git/refs/remote.py,sha256=3iSjMHPlQCUA3QacJ-CHK60yihCmGeyVnuTh-zTq7qo,2556 -git/refs/symbolic.py,sha256=nBmwXAK48w-vmFq-tAl40XlemKsPxiu7aDnyctlWA5o,29739 -git/refs/tag.py,sha256=xboM_oFCFXakpZvvb-bn4GgLcsddPvNIuEK9E3gNuNs,4273 -git/remote.py,sha256=Mp2s5t4nK_MmdTEWUhnYtr24YSysVp_humIW0d860-Q,41696 -git/repo/__init__.py,sha256=XMpdeowJRtTEd80jAcrKSQfMu2JZGMfPlpuIYHG2ZCk,80 -git/repo/__pycache__/__init__.cpython-38.pyc,, -git/repo/__pycache__/base.cpython-38.pyc,, -git/repo/__pycache__/fun.cpython-38.pyc,, -git/repo/base.py,sha256=mF3lmjtg81KQfst2RZad290fp-bwsEG28dEh7NYkmf0,51847 -git/repo/fun.py,sha256=yjsY_sna6XaUzB0ZANiIMwu97QMxWAWpWtoj4pynqg4,12713 -git/types.py,sha256=L9yBmFn6XUdV9BJgnKSwXPcYEPj9mACl7VuIi7FcNQ8,3072 -git/util.py,sha256=oEIneNH3lq2hELOtm1RJ9j7ffHEIGUpRG71UQlH5jlg,39515 diff --git a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/WHEEL b/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/WHEEL deleted file mode 100644 index e499438..0000000 --- a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: bdist_wheel (0.33.1) -Root-Is-Purelib: true -Tag: py3-none-any - diff --git a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/top_level.txt b/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/top_level.txt deleted file mode 100644 index 5664e30..0000000 --- a/.venv/lib/python3.8/site-packages/GitPython-3.1.26.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -git diff --git a/.venv/lib/python3.8/site-packages/_distutils_hack/__init__.py b/.venv/lib/python3.8/site-packages/_distutils_hack/__init__.py deleted file mode 100644 index 47ce249..0000000 --- a/.venv/lib/python3.8/site-packages/_distutils_hack/__init__.py +++ /dev/null @@ -1,128 +0,0 @@ -import sys -import os -import re -import importlib -import warnings - - -is_pypy = '__pypy__' in sys.builtin_module_names - - -warnings.filterwarnings('ignore', - '.+ distutils .+ deprecated', - DeprecationWarning) - - -def warn_distutils_present(): - if 'distutils' not in sys.modules: - return - if is_pypy and sys.version_info < (3, 7): - # PyPy for 3.6 unconditionally imports distutils, so bypass the warning - # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 - return - warnings.warn( - "Distutils was imported before Setuptools, but importing Setuptools " - "also replaces the `distutils` module in `sys.modules`. This may lead " - "to undesirable behaviors or errors. To avoid these issues, avoid " - "using distutils directly, ensure that setuptools is installed in the " - "traditional way (e.g. not an editable install), and/or make sure " - "that setuptools is always imported before distutils.") - - -def clear_distutils(): - if 'distutils' not in sys.modules: - return - warnings.warn("Setuptools is replacing distutils.") - mods = [name for name in sys.modules if re.match(r'distutils\b', name)] - for name in mods: - del sys.modules[name] - - -def enabled(): - """ - Allow selection of distutils by environment variable. - """ - which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') - return which == 'local' - - -def ensure_local_distutils(): - clear_distutils() - distutils = importlib.import_module('setuptools._distutils') - distutils.__name__ = 'distutils' - sys.modules['distutils'] = distutils - - # sanity check that submodules load as expected - core = importlib.import_module('distutils.core') - assert '_distutils' in core.__file__, core.__file__ - - -def do_override(): - """ - Ensure that the local copy of distutils is preferred over stdlib. - - See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 - for more motivation. - """ - if enabled(): - warn_distutils_present() - ensure_local_distutils() - - -class DistutilsMetaFinder: - def find_spec(self, fullname, path, target=None): - if path is not None: - return - - method_name = 'spec_for_{fullname}'.format(**locals()) - method = getattr(self, method_name, lambda: None) - return method() - - def spec_for_distutils(self): - import importlib.abc - import importlib.util - - class DistutilsLoader(importlib.abc.Loader): - - def create_module(self, spec): - return importlib.import_module('setuptools._distutils') - - def exec_module(self, module): - pass - - return importlib.util.spec_from_loader('distutils', DistutilsLoader()) - - def spec_for_pip(self): - """ - Ensure stdlib distutils when running under pip. - See pypa/pip#8761 for rationale. - """ - if self.pip_imported_during_build(): - return - clear_distutils() - self.spec_for_distutils = lambda: None - - @staticmethod - def pip_imported_during_build(): - """ - Detect if pip is being imported in a build script. Ref #2355. - """ - import traceback - return any( - frame.f_globals['__file__'].endswith('setup.py') - for frame, line in traceback.walk_stack(None) - ) - - -DISTUTILS_FINDER = DistutilsMetaFinder() - - -def add_shim(): - sys.meta_path.insert(0, DISTUTILS_FINDER) - - -def remove_shim(): - try: - sys.meta_path.remove(DISTUTILS_FINDER) - except ValueError: - pass diff --git a/.venv/lib/python3.8/site-packages/_distutils_hack/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/_distutils_hack/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index da1b6be..0000000 Binary files a/.venv/lib/python3.8/site-packages/_distutils_hack/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/_distutils_hack/__pycache__/override.cpython-38.pyc b/.venv/lib/python3.8/site-packages/_distutils_hack/__pycache__/override.cpython-38.pyc deleted file mode 100644 index 1f50baf..0000000 Binary files a/.venv/lib/python3.8/site-packages/_distutils_hack/__pycache__/override.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/_distutils_hack/override.py b/.venv/lib/python3.8/site-packages/_distutils_hack/override.py deleted file mode 100644 index 2cc433a..0000000 --- a/.venv/lib/python3.8/site-packages/_distutils_hack/override.py +++ /dev/null @@ -1 +0,0 @@ -__import__('_distutils_hack').do_override() diff --git a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/INSTALLER b/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/LICENSE b/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/LICENSE deleted file mode 100644 index d112c48..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/LICENSE +++ /dev/null @@ -1,32 +0,0 @@ -This module was extracted from the `cloud` package, developed by -PiCloud, Inc. - -Copyright (c) 2015, Cloudpickle contributors. -Copyright (c) 2012, Regents of the University of California. -Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the University of California, Berkeley nor the - names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/METADATA b/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/METADATA deleted file mode 100644 index 42d473f..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/METADATA +++ /dev/null @@ -1,185 +0,0 @@ -Metadata-Version: 2.1 -Name: cloudpickle -Version: 2.0.0 -Summary: Extended pickling support for Python objects -Home-page: https://github.com/cloudpipe/cloudpickle -Author: Cloudpipe -Author-email: cloudpipe@googlegroups.com -License: BSD 3-Clause License -Platform: UNKNOWN -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: BSD License -Classifier: Operating System :: POSIX -Classifier: Operating System :: Microsoft :: Windows -Classifier: Operating System :: MacOS :: MacOS X -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Scientific/Engineering -Classifier: Topic :: System :: Distributed Computing -Requires-Python: >=3.6 -Description-Content-Type: text/markdown -License-File: LICENSE - -# cloudpickle - -[![Automated Tests](https://github.com/cloudpipe/cloudpickle/workflows/Automated%20Tests/badge.svg?branch=master&event=push)](https://github.com/cloudpipe/cloudpickle/actions) -[![codecov.io](https://codecov.io/github/cloudpipe/cloudpickle/coverage.svg?branch=master)](https://codecov.io/github/cloudpipe/cloudpickle?branch=master) - -`cloudpickle` makes it possible to serialize Python constructs not supported -by the default `pickle` module from the Python standard library. - -`cloudpickle` is especially useful for **cluster computing** where Python -code is shipped over the network to execute on remote hosts, possibly close -to the data. - -Among other things, `cloudpickle` supports pickling for **lambda functions** -along with **functions and classes defined interactively** in the -`__main__` module (for instance in a script, a shell or a Jupyter notebook). - -Cloudpickle can only be used to send objects between the **exact same version -of Python**. - -Using `cloudpickle` for **long-term object storage is not supported and -strongly discouraged.** - -**Security notice**: one should **only load pickle data from trusted sources** as -otherwise `pickle.load` can lead to arbitrary code execution resulting in a critical -security vulnerability. - - -Installation ------------- - -The latest release of `cloudpickle` is available from -[pypi](https://pypi.python.org/pypi/cloudpickle): - - pip install cloudpickle - - -Examples --------- - -Pickling a lambda expression: - -```python ->>> import cloudpickle ->>> squared = lambda x: x ** 2 ->>> pickled_lambda = cloudpickle.dumps(squared) - ->>> import pickle ->>> new_squared = pickle.loads(pickled_lambda) ->>> new_squared(2) -4 -``` - -Pickling a function interactively defined in a Python shell session -(in the `__main__` module): - -```python ->>> CONSTANT = 42 ->>> def my_function(data: int) -> int: -... return data + CONSTANT -... ->>> pickled_function = cloudpickle.dumps(my_function) ->>> depickled_function = pickle.loads(pickled_function) ->>> depickled_function - int> ->>> depickled_function(43) -85 -``` - - -Overriding pickle's serialization mechanism for importable constructs: ----------------------------------------------------------------------- - -An important difference between `cloudpickle` and `pickle` is that -`cloudpickle` can serialize a function or class **by value**, whereas `pickle` -can only serialize it **by reference**. Serialization by reference treats -functions and classes as attributes of modules, and pickles them through -instructions that trigger the import of their module at load time. -Serialization by reference is thus limited in that it assumes that the module -containing the function or class is available/importable in the unpickling -environment. This assumption breaks when pickling constructs defined in an -interactive session, a case that is automatically detected by `cloudpickle`, -that pickles such constructs **by value**. - -Another case where the importability assumption is expected to break is when -developing a module in a distributed execution environment: the worker -processes may not have access to the said module, for example if they live on a -different machine than the process in which the module is being developed. -By itself, `cloudpickle` cannot detect such "locally importable" modules and -switch to serialization by value; instead, it relies on its default mode, -which is serialization by reference. However, since `cloudpickle 1.7.0`, one -can explicitly specify modules for which serialization by value should be used, -using the `register_pickle_by_value(module)`/`/unregister_pickle(module)` API: - -```python ->>> import cloudpickle ->>> import my_module ->>> cloudpickle.register_pickle_by_value(my_module) ->>> cloudpickle.dumps(my_module.my_function) # my_function is pickled by value ->>> cloudpickle.unregister_pickle_by_value(my_module) ->>> cloudpickle.dumps(my_module.my_function) # my_function is pickled by reference -``` - -Using this API, there is no need to re-install the new version of the module on -all the worker nodes nor to restart the workers: restarting the client Python -process with the new source code is enough. - -Note that this feature is still **experimental**, and may fail in the following -situations: - -- If the body of a function/class pickled by value contains an `import` statement: - ```python - >>> def f(): - >>> ... from another_module import g - >>> ... # calling f in the unpickling environment may fail if another_module - >>> ... # is unavailable - >>> ... return g() + 1 - ``` - -- If a function pickled by reference uses a function pickled by value during its execution. - - -Running the tests ------------------ - -- With `tox`, to test run the tests for all the supported versions of - Python and PyPy: - - pip install tox - tox - - or alternatively for a specific environment: - - tox -e py37 - - -- With `py.test` to only run the tests for your current version of - Python: - - pip install -r dev-requirements.txt - PYTHONPATH='.:tests' py.test - -History -------- - -`cloudpickle` was initially developed by [picloud.com](http://web.archive.org/web/20140721022102/http://blog.picloud.com/2013/11/17/picloud-has-joined-dropbox/) and shipped as part of -the client SDK. - -A copy of `cloudpickle.py` was included as part of PySpark, the Python -interface to [Apache Spark](https://spark.apache.org/). Davies Liu, Josh -Rosen, Thom Neale and other Apache Spark developers improved it significantly, -most notably to add support for PyPy and Python 3. - -The aim of the `cloudpickle` project is to make that work available to a wider -audience outside of the Spark ecosystem and to make it easier to improve it -further notably with the help of a dedicated non-regression test suite. - - diff --git a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/RECORD b/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/RECORD deleted file mode 100644 index 4e47c8e..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/RECORD +++ /dev/null @@ -1,14 +0,0 @@ -cloudpickle-2.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -cloudpickle-2.0.0.dist-info/LICENSE,sha256=MCnqNBc-n9wjOtMV3GsQC9HqcfUpscGvl2ZKJy_cVfU,1754 -cloudpickle-2.0.0.dist-info/METADATA,sha256=5WT_0TFE8U7sNKYP-WDcAQzAR6AYikmRdof2zJmpFbA,6888 -cloudpickle-2.0.0.dist-info/RECORD,, -cloudpickle-2.0.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92 -cloudpickle-2.0.0.dist-info/top_level.txt,sha256=zMvwgKia0TelNKJP6PaHWlxWmEm60KriUO8bb5LRZlU,12 -cloudpickle/__init__.py,sha256=wrWGX5e0KerWzSLsIlbDJ7bMVOTiuFVc1NI1w_C16f4,355 -cloudpickle/__pycache__/__init__.cpython-38.pyc,, -cloudpickle/__pycache__/cloudpickle.cpython-38.pyc,, -cloudpickle/__pycache__/cloudpickle_fast.cpython-38.pyc,, -cloudpickle/__pycache__/compat.cpython-38.pyc,, -cloudpickle/cloudpickle.py,sha256=TAHQRoYNxTfzCuWhL7VsfIt6MNHkgmN_7Pu4EmcCWsM,35841 -cloudpickle/cloudpickle_fast.py,sha256=Ziwb8Rn9LYn_HKBAmQEjjROTncVib9cOEswVe4Vh3gs,32278 -cloudpickle/compat.py,sha256=FQfviKQTNfK27JY6hXo35_EwkayliBg3o9V_htuF8e4,354 diff --git a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/WHEEL b/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/WHEEL deleted file mode 100644 index 5bad85f..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: bdist_wheel (0.37.0) -Root-Is-Purelib: true -Tag: py3-none-any - diff --git a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/top_level.txt b/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/top_level.txt deleted file mode 100644 index 37d5682..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle-2.0.0.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -cloudpickle diff --git a/.venv/lib/python3.8/site-packages/cloudpickle/__init__.py b/.venv/lib/python3.8/site-packages/cloudpickle/__init__.py deleted file mode 100644 index afb073a..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import absolute_import - - -from cloudpickle.cloudpickle import * # noqa -from cloudpickle.cloudpickle_fast import CloudPickler, dumps, dump # noqa - -# Conform to the convention used by python serialization libraries, which -# expose their Pickler subclass at top-level under the "Pickler" name. -Pickler = CloudPickler - -__version__ = '2.0.0' diff --git a/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 5b027af..0000000 Binary files a/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/cloudpickle.cpython-38.pyc b/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/cloudpickle.cpython-38.pyc deleted file mode 100644 index 21ae579..0000000 Binary files a/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/cloudpickle.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/cloudpickle_fast.cpython-38.pyc b/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/cloudpickle_fast.cpython-38.pyc deleted file mode 100644 index b845d2e..0000000 Binary files a/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/cloudpickle_fast.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/compat.cpython-38.pyc b/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/compat.cpython-38.pyc deleted file mode 100644 index 67fdc03..0000000 Binary files a/.venv/lib/python3.8/site-packages/cloudpickle/__pycache__/compat.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/cloudpickle/cloudpickle.py b/.venv/lib/python3.8/site-packages/cloudpickle/cloudpickle.py deleted file mode 100644 index 347b386..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle/cloudpickle.py +++ /dev/null @@ -1,973 +0,0 @@ -""" -This class is defined to override standard pickle functionality - -The goals of it follow: --Serialize lambdas and nested functions to compiled byte code --Deal with main module correctly --Deal with other non-serializable objects - -It does not include an unpickler, as standard python unpickling suffices. - -This module was extracted from the `cloud` package, developed by `PiCloud, Inc. -`_. - -Copyright (c) 2012, Regents of the University of California. -Copyright (c) 2009 `PiCloud, Inc. `_. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the University of California, Berkeley nor the - names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" -from __future__ import print_function - -import builtins -import dis -import opcode -import platform -import sys -import types -import weakref -import uuid -import threading -import typing -import warnings - -from .compat import pickle -from collections import OrderedDict -from typing import Generic, Union, Tuple, Callable -from pickle import _getattribute -from importlib._bootstrap import _find_spec - -try: # pragma: no branch - import typing_extensions as _typing_extensions - from typing_extensions import Literal, Final -except ImportError: - _typing_extensions = Literal = Final = None - -if sys.version_info >= (3, 5, 3): - from typing import ClassVar -else: # pragma: no cover - ClassVar = None - -if sys.version_info >= (3, 8): - from types import CellType -else: - def f(): - a = 1 - - def g(): - return a - return g - CellType = type(f().__closure__[0]) - - -# cloudpickle is meant for inter process communication: we expect all -# communicating processes to run the same Python version hence we favor -# communication speed over compatibility: -DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL - -# Names of modules whose resources should be treated as dynamic. -_PICKLE_BY_VALUE_MODULES = set() - -# Track the provenance of reconstructed dynamic classes to make it possible to -# reconstruct instances from the matching singleton class definition when -# appropriate and preserve the usual "isinstance" semantics of Python objects. -_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary() -_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary() -_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock() - -PYPY = platform.python_implementation() == "PyPy" - -builtin_code_type = None -if PYPY: - # builtin-code objects only exist in pypy - builtin_code_type = type(float.__new__.__code__) - -_extract_code_globals_cache = weakref.WeakKeyDictionary() - - -def _get_or_create_tracker_id(class_def): - with _DYNAMIC_CLASS_TRACKER_LOCK: - class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def) - if class_tracker_id is None: - class_tracker_id = uuid.uuid4().hex - _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id - _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def - return class_tracker_id - - -def _lookup_class_or_track(class_tracker_id, class_def): - if class_tracker_id is not None: - with _DYNAMIC_CLASS_TRACKER_LOCK: - class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault( - class_tracker_id, class_def) - _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id - return class_def - - -def register_pickle_by_value(module): - """Register a module to make it functions and classes picklable by value. - - By default, functions and classes that are attributes of an importable - module are to be pickled by reference, that is relying on re-importing - the attribute from the module at load time. - - If `register_pickle_by_value(module)` is called, all its functions and - classes are subsequently to be pickled by value, meaning that they can - be loaded in Python processes where the module is not importable. - - This is especially useful when developing a module in a distributed - execution environment: restarting the client Python process with the new - source code is enough: there is no need to re-install the new version - of the module on all the worker nodes nor to restart the workers. - - Note: this feature is considered experimental. See the cloudpickle - README.md file for more details and limitations. - """ - if not isinstance(module, types.ModuleType): - raise ValueError( - f"Input should be a module object, got {str(module)} instead" - ) - # In the future, cloudpickle may need a way to access any module registered - # for pickling by value in order to introspect relative imports inside - # functions pickled by value. (see - # https://github.com/cloudpipe/cloudpickle/pull/417#issuecomment-873684633). - # This access can be ensured by checking that module is present in - # sys.modules at registering time and assuming that it will still be in - # there when accessed during pickling. Another alternative would be to - # store a weakref to the module. Even though cloudpickle does not implement - # this introspection yet, in order to avoid a possible breaking change - # later, we still enforce the presence of module inside sys.modules. - if module.__name__ not in sys.modules: - raise ValueError( - f"{module} was not imported correctly, have you used an " - f"`import` statement to access it?" - ) - _PICKLE_BY_VALUE_MODULES.add(module.__name__) - - -def unregister_pickle_by_value(module): - """Unregister that the input module should be pickled by value.""" - if not isinstance(module, types.ModuleType): - raise ValueError( - f"Input should be a module object, got {str(module)} instead" - ) - if module.__name__ not in _PICKLE_BY_VALUE_MODULES: - raise ValueError(f"{module} is not registered for pickle by value") - else: - _PICKLE_BY_VALUE_MODULES.remove(module.__name__) - - -def list_registry_pickle_by_value(): - return _PICKLE_BY_VALUE_MODULES.copy() - - -def _is_registered_pickle_by_value(module): - module_name = module.__name__ - if module_name in _PICKLE_BY_VALUE_MODULES: - return True - while True: - parent_name = module_name.rsplit(".", 1)[0] - if parent_name == module_name: - break - if parent_name in _PICKLE_BY_VALUE_MODULES: - return True - module_name = parent_name - return False - - -def _whichmodule(obj, name): - """Find the module an object belongs to. - - This function differs from ``pickle.whichmodule`` in two ways: - - it does not mangle the cases where obj's module is __main__ and obj was - not found in any module. - - Errors arising during module introspection are ignored, as those errors - are considered unwanted side effects. - """ - if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa - # Workaround bug in old Python versions: prior to Python 3.7, - # T.__module__ would always be set to "typing" even when the TypeVar T - # would be defined in a different module. - if name is not None and getattr(typing, name, None) is obj: - # Built-in TypeVar defined in typing such as AnyStr - return 'typing' - else: - # User defined or third-party TypeVar: __module__ attribute is - # irrelevant, thus trigger a exhaustive search for obj in all - # modules. - module_name = None - else: - module_name = getattr(obj, '__module__', None) - - if module_name is not None: - return module_name - # Protect the iteration by using a copy of sys.modules against dynamic - # modules that trigger imports of other modules upon calls to getattr or - # other threads importing at the same time. - for module_name, module in sys.modules.copy().items(): - # Some modules such as coverage can inject non-module objects inside - # sys.modules - if ( - module_name == '__main__' or - module is None or - not isinstance(module, types.ModuleType) - ): - continue - try: - if _getattribute(module, name)[0] is obj: - return module_name - except Exception: - pass - return None - - -def _should_pickle_by_reference(obj, name=None): - """Test whether an function or a class should be pickled by reference - - Pickling by reference means by that the object (typically a function or a - class) is an attribute of a module that is assumed to be importable in the - target Python environment. Loading will therefore rely on importing the - module and then calling `getattr` on it to access the function or class. - - Pickling by reference is the only option to pickle functions and classes - in the standard library. In cloudpickle the alternative option is to - pickle by value (for instance for interactively or locally defined - functions and classes or for attributes of modules that have been - explicitly registered to be pickled by value. - """ - if isinstance(obj, types.FunctionType) or issubclass(type(obj), type): - module_and_name = _lookup_module_and_qualname(obj, name=name) - if module_and_name is None: - return False - module, name = module_and_name - return not _is_registered_pickle_by_value(module) - - elif isinstance(obj, types.ModuleType): - # We assume that sys.modules is primarily used as a cache mechanism for - # the Python import machinery. Checking if a module has been added in - # is sys.modules therefore a cheap and simple heuristic to tell us - # whether we can assume that a given module could be imported by name - # in another Python process. - if _is_registered_pickle_by_value(obj): - return False - return obj.__name__ in sys.modules - else: - raise TypeError( - "cannot check importability of {} instances".format( - type(obj).__name__) - ) - - -def _lookup_module_and_qualname(obj, name=None): - if name is None: - name = getattr(obj, '__qualname__', None) - if name is None: # pragma: no cover - # This used to be needed for Python 2.7 support but is probably not - # needed anymore. However we keep the __name__ introspection in case - # users of cloudpickle rely on this old behavior for unknown reasons. - name = getattr(obj, '__name__', None) - - module_name = _whichmodule(obj, name) - - if module_name is None: - # In this case, obj.__module__ is None AND obj was not found in any - # imported module. obj is thus treated as dynamic. - return None - - if module_name == "__main__": - return None - - # Note: if module_name is in sys.modules, the corresponding module is - # assumed importable at unpickling time. See #357 - module = sys.modules.get(module_name, None) - if module is None: - # The main reason why obj's module would not be imported is that this - # module has been dynamically created, using for example - # types.ModuleType. The other possibility is that module was removed - # from sys.modules after obj was created/imported. But this case is not - # supported, as the standard pickle does not support it either. - return None - - try: - obj2, parent = _getattribute(module, name) - except AttributeError: - # obj was not found inside the module it points to - return None - if obj2 is not obj: - return None - return module, name - - -def _extract_code_globals(co): - """ - Find all globals names read or written to by codeblock co - """ - out_names = _extract_code_globals_cache.get(co) - if out_names is None: - names = co.co_names - # We use a dict with None values instead of a set to get a - # deterministic order (assuming Python 3.6+) and avoid introducing - # non-deterministic pickle bytes as a results. - out_names = {names[oparg]: None for _, oparg in _walk_global_ops(co)} - - # Declaring a function inside another one using the "def ..." - # syntax generates a constant code object corresponding to the one - # of the nested function's As the nested function may itself need - # global variables, we need to introspect its code, extract its - # globals, (look for code object in it's co_consts attribute..) and - # add the result to code_globals - if co.co_consts: - for const in co.co_consts: - if isinstance(const, types.CodeType): - out_names.update(_extract_code_globals(const)) - - _extract_code_globals_cache[co] = out_names - - return out_names - - -def _find_imported_submodules(code, top_level_dependencies): - """ - Find currently imported submodules used by a function. - - Submodules used by a function need to be detected and referenced for the - function to work correctly at depickling time. Because submodules can be - referenced as attribute of their parent package (``package.submodule``), we - need a special introspection technique that does not rely on GLOBAL-related - opcodes to find references of them in a code object. - - Example: - ``` - import concurrent.futures - import cloudpickle - def func(): - x = concurrent.futures.ThreadPoolExecutor - if __name__ == '__main__': - cloudpickle.dumps(func) - ``` - The globals extracted by cloudpickle in the function's state include the - concurrent package, but not its submodule (here, concurrent.futures), which - is the module used by func. Find_imported_submodules will detect the usage - of concurrent.futures. Saving this module alongside with func will ensure - that calling func once depickled does not fail due to concurrent.futures - not being imported - """ - - subimports = [] - # check if any known dependency is an imported package - for x in top_level_dependencies: - if (isinstance(x, types.ModuleType) and - hasattr(x, '__package__') and x.__package__): - # check if the package has any currently loaded sub-imports - prefix = x.__name__ + '.' - # A concurrent thread could mutate sys.modules, - # make sure we iterate over a copy to avoid exceptions - for name in list(sys.modules): - # Older versions of pytest will add a "None" module to - # sys.modules. - if name is not None and name.startswith(prefix): - # check whether the function can address the sub-module - tokens = set(name[len(prefix):].split('.')) - if not tokens - set(code.co_names): - subimports.append(sys.modules[name]) - return subimports - - -def cell_set(cell, value): - """Set the value of a closure cell. - - The point of this function is to set the cell_contents attribute of a cell - after its creation. This operation is necessary in case the cell contains a - reference to the function the cell belongs to, as when calling the - function's constructor - ``f = types.FunctionType(code, globals, name, argdefs, closure)``, - closure will not be able to contain the yet-to-be-created f. - - In Python3.7, cell_contents is writeable, so setting the contents of a cell - can be done simply using - >>> cell.cell_contents = value - - In earlier Python3 versions, the cell_contents attribute of a cell is read - only, but this limitation can be worked around by leveraging the Python 3 - ``nonlocal`` keyword. - - In Python2 however, this attribute is read only, and there is no - ``nonlocal`` keyword. For this reason, we need to come up with more - complicated hacks to set this attribute. - - The chosen approach is to create a function with a STORE_DEREF opcode, - which sets the content of a closure variable. Typically: - - >>> def inner(value): - ... lambda: cell # the lambda makes cell a closure - ... cell = value # cell is a closure, so this triggers a STORE_DEREF - - (Note that in Python2, A STORE_DEREF can never be triggered from an inner - function. The function g for example here - >>> def f(var): - ... def g(): - ... var += 1 - ... return g - - will not modify the closure variable ``var```inplace, but instead try to - load a local variable var and increment it. As g does not assign the local - variable ``var`` any initial value, calling f(1)() will fail at runtime.) - - Our objective is to set the value of a given cell ``cell``. So we need to - somewhat reference our ``cell`` object into the ``inner`` function so that - this object (and not the smoke cell of the lambda function) gets affected - by the STORE_DEREF operation. - - In inner, ``cell`` is referenced as a cell variable (an enclosing variable - that is referenced by the inner function). If we create a new function - cell_set with the exact same code as ``inner``, but with ``cell`` marked as - a free variable instead, the STORE_DEREF will be applied on its closure - - ``cell``, which we can specify explicitly during construction! The new - cell_set variable thus actually sets the contents of a specified cell! - - Note: we do not make use of the ``nonlocal`` keyword to set the contents of - a cell in early python3 versions to limit possible syntax errors in case - test and checker libraries decide to parse the whole file. - """ - - if sys.version_info[:2] >= (3, 7): # pragma: no branch - cell.cell_contents = value - else: - _cell_set = types.FunctionType( - _cell_set_template_code, {}, '_cell_set', (), (cell,),) - _cell_set(value) - - -def _make_cell_set_template_code(): - def _cell_set_factory(value): - lambda: cell - cell = value - - co = _cell_set_factory.__code__ - - _cell_set_template_code = types.CodeType( - co.co_argcount, - co.co_kwonlyargcount, # Python 3 only argument - co.co_nlocals, - co.co_stacksize, - co.co_flags, - co.co_code, - co.co_consts, - co.co_names, - co.co_varnames, - co.co_filename, - co.co_name, - co.co_firstlineno, - co.co_lnotab, - co.co_cellvars, # co_freevars is initialized with co_cellvars - (), # co_cellvars is made empty - ) - return _cell_set_template_code - - -if sys.version_info[:2] < (3, 7): - _cell_set_template_code = _make_cell_set_template_code() - -# relevant opcodes -STORE_GLOBAL = opcode.opmap['STORE_GLOBAL'] -DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL'] -LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL'] -GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL) -HAVE_ARGUMENT = dis.HAVE_ARGUMENT -EXTENDED_ARG = dis.EXTENDED_ARG - - -_BUILTIN_TYPE_NAMES = {} -for k, v in types.__dict__.items(): - if type(v) is type: - _BUILTIN_TYPE_NAMES[v] = k - - -def _builtin_type(name): - if name == "ClassType": # pragma: no cover - # Backward compat to load pickle files generated with cloudpickle - # < 1.3 even if loading pickle files from older versions is not - # officially supported. - return type - return getattr(types, name) - - -def _walk_global_ops(code): - """ - Yield (opcode, argument number) tuples for all - global-referencing instructions in *code*. - """ - for instr in dis.get_instructions(code): - op = instr.opcode - if op in GLOBAL_OPS: - yield op, instr.arg - - -def _extract_class_dict(cls): - """Retrieve a copy of the dict of a class without the inherited methods""" - clsdict = dict(cls.__dict__) # copy dict proxy to a dict - if len(cls.__bases__) == 1: - inherited_dict = cls.__bases__[0].__dict__ - else: - inherited_dict = {} - for base in reversed(cls.__bases__): - inherited_dict.update(base.__dict__) - to_remove = [] - for name, value in clsdict.items(): - try: - base_value = inherited_dict[name] - if value is base_value: - to_remove.append(name) - except KeyError: - pass - for name in to_remove: - clsdict.pop(name) - return clsdict - - -if sys.version_info[:2] < (3, 7): # pragma: no branch - def _is_parametrized_type_hint(obj): - # This is very cheap but might generate false positives. So try to - # narrow it down is good as possible. - type_module = getattr(type(obj), '__module__', None) - from_typing_extensions = type_module == 'typing_extensions' - from_typing = type_module == 'typing' - - # general typing Constructs - is_typing = getattr(obj, '__origin__', None) is not None - - # typing_extensions.Literal - is_literal = ( - (getattr(obj, '__values__', None) is not None) - and from_typing_extensions - ) - - # typing_extensions.Final - is_final = ( - (getattr(obj, '__type__', None) is not None) - and from_typing_extensions - ) - - # typing.ClassVar - is_classvar = ( - (getattr(obj, '__type__', None) is not None) and from_typing - ) - - # typing.Union/Tuple for old Python 3.5 - is_union = getattr(obj, '__union_params__', None) is not None - is_tuple = getattr(obj, '__tuple_params__', None) is not None - is_callable = ( - getattr(obj, '__result__', None) is not None and - getattr(obj, '__args__', None) is not None - ) - return any((is_typing, is_literal, is_final, is_classvar, is_union, - is_tuple, is_callable)) - - def _create_parametrized_type_hint(origin, args): - return origin[args] -else: - _is_parametrized_type_hint = None - _create_parametrized_type_hint = None - - -def parametrized_type_hint_getinitargs(obj): - # The distorted type check sematic for typing construct becomes: - # ``type(obj) is type(TypeHint)``, which means "obj is a - # parametrized TypeHint" - if type(obj) is type(Literal): # pragma: no branch - initargs = (Literal, obj.__values__) - elif type(obj) is type(Final): # pragma: no branch - initargs = (Final, obj.__type__) - elif type(obj) is type(ClassVar): - initargs = (ClassVar, obj.__type__) - elif type(obj) is type(Generic): - parameters = obj.__parameters__ - if len(obj.__parameters__) > 0: - # in early Python 3.5, __parameters__ was sometimes - # preferred to __args__ - initargs = (obj.__origin__, parameters) - - else: - initargs = (obj.__origin__, obj.__args__) - elif type(obj) is type(Union): - if sys.version_info < (3, 5, 3): # pragma: no cover - initargs = (Union, obj.__union_params__) - else: - initargs = (Union, obj.__args__) - elif type(obj) is type(Tuple): - if sys.version_info < (3, 5, 3): # pragma: no cover - initargs = (Tuple, obj.__tuple_params__) - else: - initargs = (Tuple, obj.__args__) - elif type(obj) is type(Callable): - if sys.version_info < (3, 5, 3): # pragma: no cover - args = obj.__args__ - result = obj.__result__ - if args != Ellipsis: - if isinstance(args, tuple): - args = list(args) - else: - args = [args] - else: - (*args, result) = obj.__args__ - if len(args) == 1 and args[0] is Ellipsis: - args = Ellipsis - else: - args = list(args) - initargs = (Callable, (args, result)) - else: # pragma: no cover - raise pickle.PicklingError( - "Cloudpickle Error: Unknown type {}".format(type(obj)) - ) - return initargs - - -# Tornado support - -def is_tornado_coroutine(func): - """ - Return whether *func* is a Tornado coroutine function. - Running coroutines are not supported. - """ - if 'tornado.gen' not in sys.modules: - return False - gen = sys.modules['tornado.gen'] - if not hasattr(gen, "is_coroutine_function"): - # Tornado version is too old - return False - return gen.is_coroutine_function(func) - - -def _rebuild_tornado_coroutine(func): - from tornado import gen - return gen.coroutine(func) - - -# including pickles unloading functions in this namespace -load = pickle.load -loads = pickle.loads - - -def subimport(name): - # We cannot do simply: `return __import__(name)`: Indeed, if ``name`` is - # the name of a submodule, __import__ will return the top-level root module - # of this submodule. For instance, __import__('os.path') returns the `os` - # module. - __import__(name) - return sys.modules[name] - - -def dynamic_subimport(name, vars): - mod = types.ModuleType(name) - mod.__dict__.update(vars) - mod.__dict__['__builtins__'] = builtins.__dict__ - return mod - - -def _gen_ellipsis(): - return Ellipsis - - -def _gen_not_implemented(): - return NotImplemented - - -def _get_cell_contents(cell): - try: - return cell.cell_contents - except ValueError: - # sentinel used by ``_fill_function`` which will leave the cell empty - return _empty_cell_value - - -def instance(cls): - """Create a new instance of a class. - - Parameters - ---------- - cls : type - The class to create an instance of. - - Returns - ------- - instance : cls - A new instance of ``cls``. - """ - return cls() - - -@instance -class _empty_cell_value(object): - """sentinel for empty closures - """ - @classmethod - def __reduce__(cls): - return cls.__name__ - - -def _fill_function(*args): - """Fills in the rest of function data into the skeleton function object - - The skeleton itself is create by _make_skel_func(). - """ - if len(args) == 2: - func = args[0] - state = args[1] - elif len(args) == 5: - # Backwards compat for cloudpickle v0.4.0, after which the `module` - # argument was introduced - func = args[0] - keys = ['globals', 'defaults', 'dict', 'closure_values'] - state = dict(zip(keys, args[1:])) - elif len(args) == 6: - # Backwards compat for cloudpickle v0.4.1, after which the function - # state was passed as a dict to the _fill_function it-self. - func = args[0] - keys = ['globals', 'defaults', 'dict', 'module', 'closure_values'] - state = dict(zip(keys, args[1:])) - else: - raise ValueError('Unexpected _fill_value arguments: %r' % (args,)) - - # - At pickling time, any dynamic global variable used by func is - # serialized by value (in state['globals']). - # - At unpickling time, func's __globals__ attribute is initialized by - # first retrieving an empty isolated namespace that will be shared - # with other functions pickled from the same original module - # by the same CloudPickler instance and then updated with the - # content of state['globals'] to populate the shared isolated - # namespace with all the global variables that are specifically - # referenced for this function. - func.__globals__.update(state['globals']) - - func.__defaults__ = state['defaults'] - func.__dict__ = state['dict'] - if 'annotations' in state: - func.__annotations__ = state['annotations'] - if 'doc' in state: - func.__doc__ = state['doc'] - if 'name' in state: - func.__name__ = state['name'] - if 'module' in state: - func.__module__ = state['module'] - if 'qualname' in state: - func.__qualname__ = state['qualname'] - if 'kwdefaults' in state: - func.__kwdefaults__ = state['kwdefaults'] - # _cloudpickle_subimports is a set of submodules that must be loaded for - # the pickled function to work correctly at unpickling time. Now that these - # submodules are depickled (hence imported), they can be removed from the - # object's state (the object state only served as a reference holder to - # these submodules) - if '_cloudpickle_submodules' in state: - state.pop('_cloudpickle_submodules') - - cells = func.__closure__ - if cells is not None: - for cell, value in zip(cells, state['closure_values']): - if value is not _empty_cell_value: - cell_set(cell, value) - - return func - - -def _make_empty_cell(): - if False: - # trick the compiler into creating an empty cell in our lambda - cell = None - raise AssertionError('this route should not be executed') - - return (lambda: cell).__closure__[0] - - -def _make_cell(value=_empty_cell_value): - cell = _make_empty_cell() - if value is not _empty_cell_value: - cell_set(cell, value) - return cell - - -def _make_skel_func(code, cell_count, base_globals=None): - """ Creates a skeleton function object that contains just the provided - code and the correct number of cells in func_closure. All other - func attributes (e.g. func_globals) are empty. - """ - # This function is deprecated and should be removed in cloudpickle 1.7 - warnings.warn( - "A pickle file created using an old (<=1.4.1) version of cloudpickle " - "is currently being loaded. This is not supported by cloudpickle and " - "will break in cloudpickle 1.7", category=UserWarning - ) - # This is backward-compatibility code: for cloudpickle versions between - # 0.5.4 and 0.7, base_globals could be a string or None. base_globals - # should now always be a dictionary. - if base_globals is None or isinstance(base_globals, str): - base_globals = {} - - base_globals['__builtins__'] = __builtins__ - - closure = ( - tuple(_make_empty_cell() for _ in range(cell_count)) - if cell_count >= 0 else - None - ) - return types.FunctionType(code, base_globals, None, None, closure) - - -def _make_skeleton_class(type_constructor, name, bases, type_kwargs, - class_tracker_id, extra): - """Build dynamic class with an empty __dict__ to be filled once memoized - - If class_tracker_id is not None, try to lookup an existing class definition - matching that id. If none is found, track a newly reconstructed class - definition under that id so that other instances stemming from the same - class id will also reuse this class definition. - - The "extra" variable is meant to be a dict (or None) that can be used for - forward compatibility shall the need arise. - """ - skeleton_class = types.new_class( - name, bases, {'metaclass': type_constructor}, - lambda ns: ns.update(type_kwargs) - ) - return _lookup_class_or_track(class_tracker_id, skeleton_class) - - -def _rehydrate_skeleton_class(skeleton_class, class_dict): - """Put attributes from `class_dict` back on `skeleton_class`. - - See CloudPickler.save_dynamic_class for more info. - """ - registry = None - for attrname, attr in class_dict.items(): - if attrname == "_abc_impl": - registry = attr - else: - setattr(skeleton_class, attrname, attr) - if registry is not None: - for subclass in registry: - skeleton_class.register(subclass) - - return skeleton_class - - -def _make_skeleton_enum(bases, name, qualname, members, module, - class_tracker_id, extra): - """Build dynamic enum with an empty __dict__ to be filled once memoized - - The creation of the enum class is inspired by the code of - EnumMeta._create_. - - If class_tracker_id is not None, try to lookup an existing enum definition - matching that id. If none is found, track a newly reconstructed enum - definition under that id so that other instances stemming from the same - class id will also reuse this enum definition. - - The "extra" variable is meant to be a dict (or None) that can be used for - forward compatibility shall the need arise. - """ - # enums always inherit from their base Enum class at the last position in - # the list of base classes: - enum_base = bases[-1] - metacls = enum_base.__class__ - classdict = metacls.__prepare__(name, bases) - - for member_name, member_value in members.items(): - classdict[member_name] = member_value - enum_class = metacls.__new__(metacls, name, bases, classdict) - enum_class.__module__ = module - enum_class.__qualname__ = qualname - - return _lookup_class_or_track(class_tracker_id, enum_class) - - -def _make_typevar(name, bound, constraints, covariant, contravariant, - class_tracker_id): - tv = typing.TypeVar( - name, *constraints, bound=bound, - covariant=covariant, contravariant=contravariant - ) - if class_tracker_id is not None: - return _lookup_class_or_track(class_tracker_id, tv) - else: # pragma: nocover - # Only for Python 3.5.3 compat. - return tv - - -def _decompose_typevar(obj): - try: - class_tracker_id = _get_or_create_tracker_id(obj) - except TypeError: # pragma: nocover - # TypeVar instances are not weakref-able in Python 3.5.3 - class_tracker_id = None - return ( - obj.__name__, obj.__bound__, obj.__constraints__, - obj.__covariant__, obj.__contravariant__, - class_tracker_id, - ) - - -def _typevar_reduce(obj): - # TypeVar instances require the module information hence why we - # are not using the _should_pickle_by_reference directly - module_and_name = _lookup_module_and_qualname(obj, name=obj.__name__) - - if module_and_name is None: - return (_make_typevar, _decompose_typevar(obj)) - elif _is_registered_pickle_by_value(module_and_name[0]): - return (_make_typevar, _decompose_typevar(obj)) - - return (getattr, module_and_name) - - -def _get_bases(typ): - if hasattr(typ, '__orig_bases__'): - # For generic types (see PEP 560) - bases_attr = '__orig_bases__' - else: - # For regular class objects - bases_attr = '__bases__' - return getattr(typ, bases_attr) - - -def _make_dict_keys(obj, is_ordered=False): - if is_ordered: - return OrderedDict.fromkeys(obj).keys() - else: - return dict.fromkeys(obj).keys() - - -def _make_dict_values(obj, is_ordered=False): - if is_ordered: - return OrderedDict((i, _) for i, _ in enumerate(obj)).values() - else: - return {i: _ for i, _ in enumerate(obj)}.values() - - -def _make_dict_items(obj, is_ordered=False): - if is_ordered: - return OrderedDict(obj).items() - else: - return obj.items() diff --git a/.venv/lib/python3.8/site-packages/cloudpickle/cloudpickle_fast.py b/.venv/lib/python3.8/site-packages/cloudpickle/cloudpickle_fast.py deleted file mode 100644 index 6db059e..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle/cloudpickle_fast.py +++ /dev/null @@ -1,809 +0,0 @@ -""" -New, fast version of the CloudPickler. - -This new CloudPickler class can now extend the fast C Pickler instead of the -previous Python implementation of the Pickler class. Because this functionality -is only available for Python versions 3.8+, a lot of backward-compatibility -code is also removed. - -Note that the C Pickler subclassing API is CPython-specific. Therefore, some -guards present in cloudpickle.py that were written to handle PyPy specificities -are not present in cloudpickle_fast.py -""" -import _collections_abc -import abc -import copyreg -import io -import itertools -import logging -import sys -import struct -import types -import weakref -import typing - -from enum import Enum -from collections import ChainMap, OrderedDict - -from .compat import pickle, Pickler -from .cloudpickle import ( - _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, - _find_imported_submodules, _get_cell_contents, _should_pickle_by_reference, - _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, - _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, - _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, - _is_parametrized_type_hint, PYPY, cell_set, - parametrized_type_hint_getinitargs, _create_parametrized_type_hint, - builtin_code_type, - _make_dict_keys, _make_dict_values, _make_dict_items, -) - - -if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: - # Shorthands similar to pickle.dump/pickle.dumps - - def dump(obj, file, protocol=None, buffer_callback=None): - """Serialize obj as bytes streamed into file - - protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to - pickle.HIGHEST_PROTOCOL. This setting favors maximum communication - speed between processes running the same Python version. - - Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure - compatibility with older versions of Python. - """ - CloudPickler( - file, protocol=protocol, buffer_callback=buffer_callback - ).dump(obj) - - def dumps(obj, protocol=None, buffer_callback=None): - """Serialize obj as a string of bytes allocated in memory - - protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to - pickle.HIGHEST_PROTOCOL. This setting favors maximum communication - speed between processes running the same Python version. - - Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure - compatibility with older versions of Python. - """ - with io.BytesIO() as file: - cp = CloudPickler( - file, protocol=protocol, buffer_callback=buffer_callback - ) - cp.dump(obj) - return file.getvalue() - -else: - # Shorthands similar to pickle.dump/pickle.dumps - def dump(obj, file, protocol=None): - """Serialize obj as bytes streamed into file - - protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to - pickle.HIGHEST_PROTOCOL. This setting favors maximum communication - speed between processes running the same Python version. - - Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure - compatibility with older versions of Python. - """ - CloudPickler(file, protocol=protocol).dump(obj) - - def dumps(obj, protocol=None): - """Serialize obj as a string of bytes allocated in memory - - protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to - pickle.HIGHEST_PROTOCOL. This setting favors maximum communication - speed between processes running the same Python version. - - Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure - compatibility with older versions of Python. - """ - with io.BytesIO() as file: - cp = CloudPickler(file, protocol=protocol) - cp.dump(obj) - return file.getvalue() - - -load, loads = pickle.load, pickle.loads - - -# COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS -# ------------------------------------------------- - -def _class_getnewargs(obj): - type_kwargs = {} - if "__slots__" in obj.__dict__: - type_kwargs["__slots__"] = obj.__slots__ - - __dict__ = obj.__dict__.get('__dict__', None) - if isinstance(__dict__, property): - type_kwargs['__dict__'] = __dict__ - - return (type(obj), obj.__name__, _get_bases(obj), type_kwargs, - _get_or_create_tracker_id(obj), None) - - -def _enum_getnewargs(obj): - members = dict((e.name, e.value) for e in obj) - return (obj.__bases__, obj.__name__, obj.__qualname__, members, - obj.__module__, _get_or_create_tracker_id(obj), None) - - -# COLLECTION OF OBJECTS RECONSTRUCTORS -# ------------------------------------ -def _file_reconstructor(retval): - return retval - - -# COLLECTION OF OBJECTS STATE GETTERS -# ----------------------------------- -def _function_getstate(func): - # - Put func's dynamic attributes (stored in func.__dict__) in state. These - # attributes will be restored at unpickling time using - # f.__dict__.update(state) - # - Put func's members into slotstate. Such attributes will be restored at - # unpickling time by iterating over slotstate and calling setattr(func, - # slotname, slotvalue) - slotstate = { - "__name__": func.__name__, - "__qualname__": func.__qualname__, - "__annotations__": func.__annotations__, - "__kwdefaults__": func.__kwdefaults__, - "__defaults__": func.__defaults__, - "__module__": func.__module__, - "__doc__": func.__doc__, - "__closure__": func.__closure__, - } - - f_globals_ref = _extract_code_globals(func.__code__) - f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in - func.__globals__} - - closure_values = ( - list(map(_get_cell_contents, func.__closure__)) - if func.__closure__ is not None else () - ) - - # Extract currently-imported submodules used by func. Storing these modules - # in a smoke _cloudpickle_subimports attribute of the object's state will - # trigger the side effect of importing these modules at unpickling time - # (which is necessary for func to work correctly once depickled) - slotstate["_cloudpickle_submodules"] = _find_imported_submodules( - func.__code__, itertools.chain(f_globals.values(), closure_values)) - slotstate["__globals__"] = f_globals - - state = func.__dict__ - return state, slotstate - - -def _class_getstate(obj): - clsdict = _extract_class_dict(obj) - clsdict.pop('__weakref__', None) - - if issubclass(type(obj), abc.ABCMeta): - # If obj is an instance of an ABCMeta subclass, don't pickle the - # cache/negative caches populated during isinstance/issubclass - # checks, but pickle the list of registered subclasses of obj. - clsdict.pop('_abc_cache', None) - clsdict.pop('_abc_negative_cache', None) - clsdict.pop('_abc_negative_cache_version', None) - registry = clsdict.pop('_abc_registry', None) - if registry is None: - # in Python3.7+, the abc caches and registered subclasses of a - # class are bundled into the single _abc_impl attribute - clsdict.pop('_abc_impl', None) - (registry, _, _, _) = abc._get_dump(obj) - - clsdict["_abc_impl"] = [subclass_weakref() - for subclass_weakref in registry] - else: - # In the above if clause, registry is a set of weakrefs -- in - # this case, registry is a WeakSet - clsdict["_abc_impl"] = [type_ for type_ in registry] - - if "__slots__" in clsdict: - # pickle string length optimization: member descriptors of obj are - # created automatically from obj's __slots__ attribute, no need to - # save them in obj's state - if isinstance(obj.__slots__, str): - clsdict.pop(obj.__slots__) - else: - for k in obj.__slots__: - clsdict.pop(k, None) - - clsdict.pop('__dict__', None) # unpicklable property object - - return (clsdict, {}) - - -def _enum_getstate(obj): - clsdict, slotstate = _class_getstate(obj) - - members = dict((e.name, e.value) for e in obj) - # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: - # Those attributes are already handled by the metaclass. - for attrname in ["_generate_next_value_", "_member_names_", - "_member_map_", "_member_type_", - "_value2member_map_"]: - clsdict.pop(attrname, None) - for member in members: - clsdict.pop(member) - # Special handling of Enum subclasses - return clsdict, slotstate - - -# COLLECTIONS OF OBJECTS REDUCERS -# ------------------------------- -# A reducer is a function taking a single argument (obj), and that returns a -# tuple with all the necessary data to re-construct obj. Apart from a few -# exceptions (list, dict, bytes, int, etc.), a reducer is necessary to -# correctly pickle an object. -# While many built-in objects (Exceptions objects, instances of the "object" -# class, etc), are shipped with their own built-in reducer (invoked using -# obj.__reduce__), some do not. The following methods were created to "fill -# these holes". - -def _code_reduce(obj): - """codeobject reducer""" - if hasattr(obj, "co_linetable"): # pragma: no branch - # Python 3.10 and later: obj.co_lnotab is deprecated and constructor - # expects obj.co_linetable instead. - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_linetable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_posonlyargcount"): - # Backward compat for 3.9 and older - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) - else: - # Backward compat for even older versions of Python - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - obj.co_names, obj.co_varnames, obj.co_filename, - obj.co_name, obj.co_firstlineno, obj.co_lnotab, - obj.co_freevars, obj.co_cellvars - ) - return types.CodeType, args - - -def _cell_reduce(obj): - """Cell (containing values of a function's free variables) reducer""" - try: - obj.cell_contents - except ValueError: # cell is empty - return _make_empty_cell, () - else: - return _make_cell, (obj.cell_contents, ) - - -def _classmethod_reduce(obj): - orig_func = obj.__func__ - return type(obj), (orig_func,) - - -def _file_reduce(obj): - """Save a file""" - import io - - if not hasattr(obj, "name") or not hasattr(obj, "mode"): - raise pickle.PicklingError( - "Cannot pickle files that do not map to an actual file" - ) - if obj is sys.stdout: - return getattr, (sys, "stdout") - if obj is sys.stderr: - return getattr, (sys, "stderr") - if obj is sys.stdin: - raise pickle.PicklingError("Cannot pickle standard input") - if obj.closed: - raise pickle.PicklingError("Cannot pickle closed files") - if hasattr(obj, "isatty") and obj.isatty(): - raise pickle.PicklingError( - "Cannot pickle files that map to tty objects" - ) - if "r" not in obj.mode and "+" not in obj.mode: - raise pickle.PicklingError( - "Cannot pickle files that are not opened for reading: %s" - % obj.mode - ) - - name = obj.name - - retval = io.StringIO() - - try: - # Read the whole file - curloc = obj.tell() - obj.seek(0) - contents = obj.read() - obj.seek(curloc) - except IOError as e: - raise pickle.PicklingError( - "Cannot pickle file %s as it cannot be read" % name - ) from e - retval.write(contents) - retval.seek(curloc) - - retval.name = name - return _file_reconstructor, (retval,) - - -def _getset_descriptor_reduce(obj): - return getattr, (obj.__objclass__, obj.__name__) - - -def _mappingproxy_reduce(obj): - return types.MappingProxyType, (dict(obj),) - - -def _memoryview_reduce(obj): - return bytes, (obj.tobytes(),) - - -def _module_reduce(obj): - if _should_pickle_by_reference(obj): - return subimport, (obj.__name__,) - else: - # Some external libraries can populate the "__builtins__" entry of a - # module's `__dict__` with unpicklable objects (see #316). For that - # reason, we do not attempt to pickle the "__builtins__" entry, and - # restore a default value for it at unpickling time. - state = obj.__dict__.copy() - state.pop('__builtins__', None) - return dynamic_subimport, (obj.__name__, state) - - -def _method_reduce(obj): - return (types.MethodType, (obj.__func__, obj.__self__)) - - -def _logger_reduce(obj): - return logging.getLogger, (obj.name,) - - -def _root_logger_reduce(obj): - return logging.getLogger, () - - -def _property_reduce(obj): - return property, (obj.fget, obj.fset, obj.fdel, obj.__doc__) - - -def _weakset_reduce(obj): - return weakref.WeakSet, (list(obj),) - - -def _dynamic_class_reduce(obj): - """ - Save a class that can't be stored as module global. - - This method is used to serialize classes that are defined inside - functions, or that otherwise can't be serialized as attribute lookups - from global modules. - """ - if Enum is not None and issubclass(obj, Enum): - return ( - _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj), - None, None, _class_setstate - ) - else: - return ( - _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj), - None, None, _class_setstate - ) - - -def _class_reduce(obj): - """Select the reducer depending on the dynamic nature of the class obj""" - if obj is type(None): # noqa - return type, (None,) - elif obj is type(Ellipsis): - return type, (Ellipsis,) - elif obj is type(NotImplemented): - return type, (NotImplemented,) - elif obj in _BUILTIN_TYPE_NAMES: - return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],) - elif not _should_pickle_by_reference(obj): - return _dynamic_class_reduce(obj) - return NotImplemented - - -def _dict_keys_reduce(obj): - # Safer not to ship the full dict as sending the rest might - # be unintended and could potentially cause leaking of - # sensitive information - return _make_dict_keys, (list(obj), ) - - -def _dict_values_reduce(obj): - # Safer not to ship the full dict as sending the rest might - # be unintended and could potentially cause leaking of - # sensitive information - return _make_dict_values, (list(obj), ) - - -def _dict_items_reduce(obj): - return _make_dict_items, (dict(obj), ) - - -def _odict_keys_reduce(obj): - # Safer not to ship the full dict as sending the rest might - # be unintended and could potentially cause leaking of - # sensitive information - return _make_dict_keys, (list(obj), True) - - -def _odict_values_reduce(obj): - # Safer not to ship the full dict as sending the rest might - # be unintended and could potentially cause leaking of - # sensitive information - return _make_dict_values, (list(obj), True) - - -def _odict_items_reduce(obj): - return _make_dict_items, (dict(obj), True) - - -# COLLECTIONS OF OBJECTS STATE SETTERS -# ------------------------------------ -# state setters are called at unpickling time, once the object is created and -# it has to be updated to how it was at unpickling time. - - -def _function_setstate(obj, state): - """Update the state of a dynamic function. - - As __closure__ and __globals__ are readonly attributes of a function, we - cannot rely on the native setstate routine of pickle.load_build, that calls - setattr on items of the slotstate. Instead, we have to modify them inplace. - """ - state, slotstate = state - obj.__dict__.update(state) - - obj_globals = slotstate.pop("__globals__") - obj_closure = slotstate.pop("__closure__") - # _cloudpickle_subimports is a set of submodules that must be loaded for - # the pickled function to work correctly at unpickling time. Now that these - # submodules are depickled (hence imported), they can be removed from the - # object's state (the object state only served as a reference holder to - # these submodules) - slotstate.pop("_cloudpickle_submodules") - - obj.__globals__.update(obj_globals) - obj.__globals__["__builtins__"] = __builtins__ - - if obj_closure is not None: - for i, cell in enumerate(obj_closure): - try: - value = cell.cell_contents - except ValueError: # cell is empty - continue - cell_set(obj.__closure__[i], value) - - for k, v in slotstate.items(): - setattr(obj, k, v) - - -def _class_setstate(obj, state): - state, slotstate = state - registry = None - for attrname, attr in state.items(): - if attrname == "_abc_impl": - registry = attr - else: - setattr(obj, attrname, attr) - if registry is not None: - for subclass in registry: - obj.register(subclass) - - return obj - - -class CloudPickler(Pickler): - # set of reducers defined and used by cloudpickle (private) - _dispatch_table = {} - _dispatch_table[classmethod] = _classmethod_reduce - _dispatch_table[io.TextIOWrapper] = _file_reduce - _dispatch_table[logging.Logger] = _logger_reduce - _dispatch_table[logging.RootLogger] = _root_logger_reduce - _dispatch_table[memoryview] = _memoryview_reduce - _dispatch_table[property] = _property_reduce - _dispatch_table[staticmethod] = _classmethod_reduce - _dispatch_table[CellType] = _cell_reduce - _dispatch_table[types.CodeType] = _code_reduce - _dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce - _dispatch_table[types.ModuleType] = _module_reduce - _dispatch_table[types.MethodType] = _method_reduce - _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce - _dispatch_table[weakref.WeakSet] = _weakset_reduce - _dispatch_table[typing.TypeVar] = _typevar_reduce - _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce - _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce - _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce - _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce - _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce - _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce - - - dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) - - # function reducers are defined as instance methods of CloudPickler - # objects, as they rely on a CloudPickler attribute (globals_ref) - def _dynamic_function_reduce(self, func): - """Reduce a function that is not pickleable via attribute lookup.""" - newargs = self._function_getnewargs(func) - state = _function_getstate(func) - return (types.FunctionType, newargs, state, None, None, - _function_setstate) - - def _function_reduce(self, obj): - """Reducer for function objects. - - If obj is a top-level attribute of a file-backed module, this - reducer returns NotImplemented, making the CloudPickler fallback to - traditional _pickle.Pickler routines to save obj. Otherwise, it reduces - obj using a custom cloudpickle reducer designed specifically to handle - dynamic functions. - - As opposed to cloudpickle.py, There no special handling for builtin - pypy functions because cloudpickle_fast is CPython-specific. - """ - if _should_pickle_by_reference(obj): - return NotImplemented - else: - return self._dynamic_function_reduce(obj) - - def _function_getnewargs(self, func): - code = func.__code__ - - # base_globals represents the future global namespace of func at - # unpickling time. Looking it up and storing it in - # CloudpiPickler.globals_ref allow functions sharing the same globals - # at pickling time to also share them once unpickled, at one condition: - # since globals_ref is an attribute of a CloudPickler instance, and - # that a new CloudPickler is created each time pickle.dump or - # pickle.dumps is called, functions also need to be saved within the - # same invocation of cloudpickle.dump/cloudpickle.dumps (for example: - # cloudpickle.dumps([f1, f2])). There is no such limitation when using - # CloudPickler.dump, as long as the multiple invocations are bound to - # the same CloudPickler. - base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) - - if base_globals == {}: - # Add module attributes used to resolve relative imports - # instructions inside func. - for k in ["__package__", "__name__", "__path__", "__file__"]: - if k in func.__globals__: - base_globals[k] = func.__globals__[k] - - # Do not bind the free variables before the function is created to - # avoid infinite recursion. - if func.__closure__ is None: - closure = None - else: - closure = tuple( - _make_empty_cell() for _ in range(len(code.co_freevars))) - - return code, base_globals, None, None, closure - - def dump(self, obj): - try: - return Pickler.dump(self, obj) - except RuntimeError as e: - if "recursion" in e.args[0]: - msg = ( - "Could not pickle object as excessively deep recursion " - "required." - ) - raise pickle.PicklingError(msg) from e - else: - raise - - if pickle.HIGHEST_PROTOCOL >= 5: - # `CloudPickler.dispatch` is only left for backward compatibility - note - # that when using protocol 5, `CloudPickler.dispatch` is not an - # extension of `Pickler.dispatch` dictionary, because CloudPickler - # subclasses the C-implemented Pickler, which does not expose a - # `dispatch` attribute. Earlier versions of the protocol 5 CloudPickler - # used `CloudPickler.dispatch` as a class-level attribute storing all - # reducers implemented by cloudpickle, but the attribute name was not a - # great choice given the meaning of `CloudPickler.dispatch` when - # `CloudPickler` extends the pure-python pickler. - dispatch = dispatch_table - - # Implementation of the reducer_override callback, in order to - # efficiently serialize dynamic functions and classes by subclassing - # the C-implemented Pickler. - # TODO: decorrelate reducer_override (which is tied to CPython's - # implementation - would it make sense to backport it to pypy? - and - # pickle's protocol 5 which is implementation agnostic. Currently, the - # availability of both notions coincide on CPython's pickle and the - # pickle5 backport, but it may not be the case anymore when pypy - # implements protocol 5 - def __init__(self, file, protocol=None, buffer_callback=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__( - self, file, protocol=protocol, buffer_callback=buffer_callback - ) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share - # their global namespace at unpickling time. - self.globals_ref = {} - self.proto = int(protocol) - - def reducer_override(self, obj): - """Type-agnostic reducing callback for function and classes. - - For performance reasons, subclasses of the C _pickle.Pickler class - cannot register custom reducers for functions and classes in the - dispatch_table. Reducer for such types must instead implemented in - the special reducer_override method. - - Note that method will be called for any object except a few - builtin-types (int, lists, dicts etc.), which differs from reducers - in the Pickler's dispatch_table, each of them being invoked for - objects of a specific type only. - - This property comes in handy for classes: although most classes are - instances of the ``type`` metaclass, some of them can be instances - of other custom metaclasses (such as enum.EnumMeta for example). In - particular, the metaclass will likely not be known in advance, and - thus cannot be special-cased using an entry in the dispatch_table. - reducer_override, among other things, allows us to register a - reducer that will be called for any class, independently of its - type. - - - Notes: - - * reducer_override has the priority over dispatch_table-registered - reducers. - * reducer_override can be used to fix other limitations of - cloudpickle for other types that suffered from type-specific - reducers, such as Exceptions. See - https://github.com/cloudpipe/cloudpickle/issues/248 - """ - if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch - return ( - _create_parametrized_type_hint, - parametrized_type_hint_getinitargs(obj) - ) - t = type(obj) - try: - is_anyclass = issubclass(t, type) - except TypeError: # t is not a class (old Boost; see SF #502085) - is_anyclass = False - - if is_anyclass: - return _class_reduce(obj) - elif isinstance(obj, types.FunctionType): - return self._function_reduce(obj) - else: - # fallback to save_global, including the Pickler's - # dispatch_table - return NotImplemented - - else: - # When reducer_override is not available, hack the pure-Python - # Pickler's types.FunctionType and type savers. Note: the type saver - # must override Pickler.save_global, because pickle.py contains a - # hard-coded call to save_global when pickling meta-classes. - dispatch = Pickler.dispatch.copy() - - def __init__(self, file, protocol=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__(self, file, protocol=protocol) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share - # their global namespace at unpickling time. - self.globals_ref = {} - assert hasattr(self, 'proto') - - def _save_reduce_pickle5(self, func, args, state=None, listitems=None, - dictitems=None, state_setter=None, obj=None): - save = self.save - write = self.write - self.save_reduce( - func, args, state=None, listitems=listitems, - dictitems=dictitems, obj=obj - ) - # backport of the Python 3.8 state_setter pickle operations - save(state_setter) - save(obj) # simple BINGET opcode as obj is already memoized. - save(state) - write(pickle.TUPLE2) - # Trigger a state_setter(obj, state) function call. - write(pickle.REDUCE) - # The purpose of state_setter is to carry-out an - # inplace modification of obj. We do not care about what the - # method might return, so its output is eventually removed from - # the stack. - write(pickle.POP) - - def save_global(self, obj, name=None, pack=struct.pack): - """ - Save a "global". - - The name of this method is somewhat misleading: all types get - dispatched here. - """ - if obj is type(None): # noqa - return self.save_reduce(type, (None,), obj=obj) - elif obj is type(Ellipsis): - return self.save_reduce(type, (Ellipsis,), obj=obj) - elif obj is type(NotImplemented): - return self.save_reduce(type, (NotImplemented,), obj=obj) - elif obj in _BUILTIN_TYPE_NAMES: - return self.save_reduce( - _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) - - if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch - # Parametrized typing constructs in Python < 3.7 are not - # compatible with type checks and ``isinstance`` semantics. For - # this reason, it is easier to detect them using a - # duck-typing-based check (``_is_parametrized_type_hint``) than - # to populate the Pickler's dispatch with type-specific savers. - self.save_reduce( - _create_parametrized_type_hint, - parametrized_type_hint_getinitargs(obj), - obj=obj - ) - elif name is not None: - Pickler.save_global(self, obj, name=name) - elif not _should_pickle_by_reference(obj, name=name): - self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) - else: - Pickler.save_global(self, obj, name=name) - dispatch[type] = save_global - - def save_function(self, obj, name=None): - """ Registered with the dispatch to handle all function types. - - Determines what kind of function obj is (e.g. lambda, defined at - interactive prompt, etc) and handles the pickling appropriately. - """ - if _should_pickle_by_reference(obj, name=name): - return Pickler.save_global(self, obj, name=name) - elif PYPY and isinstance(obj.__code__, builtin_code_type): - return self.save_pypy_builtin_func(obj) - else: - return self._save_reduce_pickle5( - *self._dynamic_function_reduce(obj), obj=obj - ) - - def save_pypy_builtin_func(self, obj): - """Save pypy equivalent of builtin functions. - PyPy does not have the concept of builtin-functions. Instead, - builtin-functions are simple function instances, but with a - builtin-code attribute. - Most of the time, builtin functions should be pickled by attribute. - But PyPy has flaky support for __qualname__, so some builtin - functions such as float.__new__ will be classified as dynamic. For - this reason only, we created this special routine. Because - builtin-functions are not expected to have closure or globals, - there is no additional hack (compared the one already implemented - in pickle) to protect ourselves from reference cycles. A simple - (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note - also that PyPy improved their support for __qualname__ in v3.6, so - this routing should be removed when cloudpickle supports only PyPy - 3.6 and later. - """ - rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, - obj.__defaults__, obj.__closure__), - obj.__dict__) - self.save_reduce(*rv, obj=obj) - - dispatch[types.FunctionType] = save_function diff --git a/.venv/lib/python3.8/site-packages/cloudpickle/compat.py b/.venv/lib/python3.8/site-packages/cloudpickle/compat.py deleted file mode 100644 index afa285f..0000000 --- a/.venv/lib/python3.8/site-packages/cloudpickle/compat.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys - - -if sys.version_info < (3, 8): - try: - import pickle5 as pickle # noqa: F401 - from pickle5 import Pickler # noqa: F401 - except ImportError: - import pickle # noqa: F401 - from pickle import _Pickler as Pickler # noqa: F401 -else: - import pickle # noqa: F401 - from _pickle import Pickler # noqa: F401 diff --git a/.venv/lib/python3.8/site-packages/distutils-precedence.pth b/.venv/lib/python3.8/site-packages/distutils-precedence.pth deleted file mode 100644 index 6de4198..0000000 --- a/.venv/lib/python3.8/site-packages/distutils-precedence.pth +++ /dev/null @@ -1 +0,0 @@ -import os; var = 'SETUPTOOLS_USE_DISTUTILS'; enabled = os.environ.get(var, 'stdlib') == 'local'; enabled and __import__('_distutils_hack').add_shim(); diff --git a/.venv/lib/python3.8/site-packages/flor/__init__.py b/.venv/lib/python3.8/site-packages/flor/__init__.py deleted file mode 100644 index f6d56e9..0000000 --- a/.venv/lib/python3.8/site-packages/flor/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from . import flags -from .iterator import it -from .skipblock import SkipBlock -from .logger import Logger -from .pin import pin - -flags.Parser.parse() - - -__all__ = ["flags", "it", "pin", "SkipBlock", "Logger"] diff --git a/.venv/lib/python3.8/site-packages/flor/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index b6b9567..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/__pycache__/constants.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/__pycache__/constants.cpython-38.pyc deleted file mode 100644 index 3a83f41..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/__pycache__/constants.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/__pycache__/flags.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/__pycache__/flags.cpython-38.pyc deleted file mode 100644 index 8636d74..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/__pycache__/flags.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/__pycache__/iterator.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/__pycache__/iterator.cpython-38.pyc deleted file mode 100644 index 13dcc0c..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/__pycache__/iterator.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/__pycache__/pin.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/__pycache__/pin.cpython-38.pyc deleted file mode 100644 index 6d879cd..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/__pycache__/pin.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/constants.py b/.venv/lib/python3.8/site-packages/flor/constants.py deleted file mode 100644 index ab425e3..0000000 --- a/.venv/lib/python3.8/site-packages/flor/constants.py +++ /dev/null @@ -1,15 +0,0 @@ -from dataclasses import dataclass -from enum import Enum - -REPLAY_MODE = Enum("REPLAY_MODE", "weak strong") -SHADOW_BRANCH_PREFIX = "flor.shadow" -FLORFILE = ".replay.json" - - -@dataclass -class REPLAY_PARALLEL: - pid: int - ngpus: int - - -__all__ = ["REPLAY_MODE", "SHADOW_BRANCH_PREFIX", "FLORFILE", "REPLAY_PARALLEL"] diff --git a/.venv/lib/python3.8/site-packages/flor/flags.py b/.venv/lib/python3.8/site-packages/flor/flags.py deleted file mode 100644 index e89053d..0000000 --- a/.venv/lib/python3.8/site-packages/flor/flags.py +++ /dev/null @@ -1,168 +0,0 @@ -import json -from typing import Dict, Optional, Tuple -from flor import shelf - -import sys -from pathlib import PurePath, Path -from .constants import * -from .pin import kvs - - -NAME: Optional[str] = None -REPLAY: bool = False -INDEX: Optional[PurePath] = None -MODE: REPLAY_MODE = REPLAY_MODE.weak -PID: REPLAY_PARALLEL = REPLAY_PARALLEL(1, 1) -EPSILON: float = 1 / 15 -RESUMING: bool = False - - -""" ---flor NAME ls[EPSILON] ---replay_flor [weak | strong] [i/n] -""" - - -def set_REPLAY( - name: str, - index: Optional[str] = None, - mode: Optional[str] = None, - pid: Optional[Tuple[int, int]] = None, -): - """ - When set: enables FLOR REPLAY - """ - global NAME, REPLAY, INDEX, MODE, PID - NAME = name - REPLAY = True - if index is not None: - assert isinstance(index, str) - assert PurePath(index).suffix == ".json" - assert shelf.verify(PurePath(index)) - INDEX = PurePath(index) - if mode is not None: - MODE = REPLAY_MODE[mode] - if pid is not None: - assert isinstance(pid, tuple) and len(pid) == 2 - p, n = pid - assert isinstance(p, int) and isinstance(n, int) - assert p >= 1 - assert n >= 1 - assert p <= n - PID = REPLAY_PARALLEL(*pid) - - -class Parser: - """ - --flor NAME [EPSILON] - --replay_flor [weak | strong] [i/n] - """ - - @staticmethod - def _parse_name(): - assert ( - "--replay_flor" not in sys.argv - ), "Pick at most one of `--flor` or `--replay_flor` but not both" - global NAME, EPSILON - flor_flags = [] - feeding = False - for _ in range(len(sys.argv)): - arg = sys.argv.pop(0) - - if arg == "--flor": - feeding = True - elif arg[0] == "-": - feeding = False - - if feeding: - flor_flags.append(arg) - else: - sys.argv.append(arg) - - assert len(flor_flags) <= 3 - if flor_flags: - assert flor_flags.pop(0) == "--flor" - assert ( - flor_flags or Path(FLORFILE).exists() - ), "Missing NAME argument in --flor NAME" - for flag in flor_flags: - if flag[0:2] == "0.": - EPSILON = float(flag) - else: - NAME = flag - if NAME is None: - try: - with open(FLORFILE, "r", encoding="utf-8") as f: - d = json.load(f) - except FileNotFoundError: - print("No replay file, did you record first?") - raise - assert "NAME" in d - NAME = d["NAME"] - - @staticmethod - def _parse_replay(): - assert ( - "--flor" not in sys.argv - ), "Pick at most one of `--flor` or `--replay_flor` but not both" - try: - with open(FLORFILE, "r", encoding="utf-8") as f: - d = json.load(f) - except FileNotFoundError: - print("No replay file, did you record first?") - raise - assert "NAME" in d, "check your `.replay.json` file. Missing name." - assert "MEMO" in d, "check your `.replay.json` file. Missing memo." - if "KVS" in d: - kvs.update(d["KVS"]) - flor_flags = [] - feeding = False - for _ in range(len(sys.argv)): - arg = sys.argv.pop(0) - - if arg == "--replay_flor": - feeding = True - elif arg[0] == "-": - feeding = False - - if feeding: - flor_flags.append(arg) - else: - sys.argv.append(arg) - - assert len(flor_flags) <= 3 - if flor_flags: - assert flor_flags.pop(0) == "--replay_flor" - mode = None - pid = None - for flor_flag in flor_flags: - if flor_flag in [each.name for each in REPLAY_MODE]: - mode = flor_flag - elif len(flor_flag.split("/")) == 2: - p, n = flor_flag.split("/") - pid = int(p), int(n) - else: - raise RuntimeError( - "Invalid argument passed to --replay_flor" - + "[weak | strong] [I/N]" - ) - set_REPLAY(d["NAME"], index=d["MEMO"], mode=mode, pid=pid) - - @staticmethod - def parse(): - if "--flor" in sys.argv: - Parser._parse_name() - elif "--replay_flor" in sys.argv: - Parser._parse_replay() - - -__all__ = [ - "NAME", - "REPLAY", - "INDEX", - "MODE", - "PID", - "EPSILON", - "set_REPLAY", - "Parser", -] diff --git a/.venv/lib/python3.8/site-packages/flor/iterator.py b/.venv/lib/python3.8/site-packages/flor/iterator.py deleted file mode 100644 index af65af4..0000000 --- a/.venv/lib/python3.8/site-packages/flor/iterator.py +++ /dev/null @@ -1,119 +0,0 @@ -import json -import os -from typing import Iterable, List, Union - -from git.exc import InvalidGitRepositoryError -from git.repo import Repo - -from . import flags, shelf -from .skipblock import SkipBlock - -from .constants import * -from .pin import kvs - - -def it(value: Union[Iterable, bool]): - """ - Main loop wrapper - :param value: - Iterable when iterating over a for loop - Bool when looping with while - """ - assert isinstance(value, (Iterable, bool)) - if flags.NAME is None: - if isinstance(value, bool): - return value - else: - assert isinstance(value, Iterable) - for each in value: - yield each - return - - _deferred_init() - - if not flags.REPLAY: - # Record mode - if isinstance(value, bool): - if not value: - _close_record() - return value - else: - for each in value: - yield each - _close_record() - else: - # Replay mode - segment = SkipBlock.journal.get_segment_window() - for capsule in segment: - flags.RESUMING = capsule.init_only - if isinstance(value, bool): - yield True - else: - assert isinstance(value, Iterable) - if flags.RESUMING: - if capsule.epoch is None: - continue - else: - assert hasattr( - value, "__getitem__" - ), "TODO: Implement next() calls to consume iterator" - yield value[capsule.epoch] # type: ignore - else: - assert capsule.epoch is not None - assert hasattr( - value, "__getitem__" - ), "TODO: Implement next() calls to consume iterator" - yield value[capsule.epoch] # type: ignore - - -def _deferred_init(_nil=[]): - """ - At most once execution - """ - if not _nil: - assert flags.NAME is not None - if not flags.REPLAY: - repo = Repo() - assert ( - SHADOW_BRANCH_PREFIX - == repo.active_branch.name[0 : len(SHADOW_BRANCH_PREFIX)] - ), f"Please run FLOR from a shadow branch (branch name: `{SHADOW_BRANCH_PREFIX}.[...]`)\nso we may commit dirty pages automatically" - shelf.mk_job(flags.NAME) - SkipBlock.bind() - if flags.REPLAY: - SkipBlock.journal.read() - else: - SkipBlock.logger.set_path(shelf.get_index()) - assert SkipBlock.logger.path is not None - _nil.append(True) - - -def _close_record(): - commit_sha = _save_run() - SkipBlock.logger.append(SkipBlock.journal.get_eof(commit_sha)) - SkipBlock.logger.close() - return commit_sha, SkipBlock.logger.path - - -def _save_run() -> str: - assert SkipBlock.logger.path is not None - repo = Repo() - _write_replay_file() - repo.git.add("-A") - commit = repo.index.commit( - f"{repo.active_branch.name}@{flags.NAME}::{SkipBlock.logger.path.name}" - ) - commit_sha = commit.hexsha - return commit_sha - - -def _write_replay_file(): - d = {} - d["NAME"] = flags.NAME - d["MEMO"] = str(SkipBlock.logger.path) - d["KVS"] = kvs - with open(FLORFILE, "w", encoding="utf-8") as f: - json.dump(d, f, ensure_ascii=False, indent=4) - - -__all__ = ["it"] diff --git a/.venv/lib/python3.8/site-packages/flor/journal/__init__.py b/.venv/lib/python3.8/site-packages/flor/journal/__init__.py deleted file mode 100644 index 9626a83..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .journal import Journal diff --git a/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 5ab1731..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/file.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/file.cpython-38.pyc deleted file mode 100644 index 20f66a2..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/file.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/journal.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/journal.cpython-38.pyc deleted file mode 100644 index e543c19..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/__pycache__/journal.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/__init__.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/__init__.py deleted file mode 100644 index 3fff1a8..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .constants import LBRACKET, RBRACKET -from .data import * -from .metadata import * -from .constructor import make_entry diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 6f15aac..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/abstract.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/abstract.cpython-38.pyc deleted file mode 100644 index d8b5ee1..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/abstract.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/constants.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/constants.cpython-38.pyc deleted file mode 100644 index 1303c22..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/constants.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/constructor.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/constructor.cpython-38.pyc deleted file mode 100644 index f7f8b36..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/__pycache__/constructor.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/abstract.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/abstract.py deleted file mode 100644 index 23f5242..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/abstract.py +++ /dev/null @@ -1,37 +0,0 @@ -from .constants import * -from abc import ABC, abstractmethod - - -class Entry(ABC): - next_lsn = 0 - - def __init__(self, sk, gk): - self.sk = sk - self.gk = gk - self.lsn = Entry.next_lsn - Entry.next_lsn += 1 - - def jsonify(self): - d = dict() - d[STATIC_KEY] = str(self.sk) - d[GLOBAL_KEY] = int(self.gk) - d[GLOBAL_LSN] = int(self.lsn) - return d - - @abstractmethod - def is_left(self): - ... - - @abstractmethod - def is_right(self): - ... - - @staticmethod - @abstractmethod - def is_superclass(json_dict: dict): - ... - - @classmethod - @abstractmethod - def cons(cls, json_dict: dict): - ... diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/constants.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/constants.py deleted file mode 100644 index b8fdf53..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/constants.py +++ /dev/null @@ -1,12 +0,0 @@ -STATIC_KEY = "static_key" -GLOBAL_KEY = "global_key" -GLOBAL_LSN = "global_lsn" -VAL = "value" -REF = "ref" -LBRACKET = "LBRACKET" -RBRACKET = "RBRACKET" -EOF_NAME = "EOF" -METADATA = "metadata" -SPARSE_CHECKPOINTS = "sparse_checkpoints" -ITERATIONS_COUNT = "iterations_count" -PKL_SFX = ".pkl" diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/constructor.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/constructor.py deleted file mode 100644 index f4e2b88..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/constructor.py +++ /dev/null @@ -1,22 +0,0 @@ -from .constants import * -from .data import * -from .metadata import * - -from typing import Union - - -def make_entry(json_dict: dict) -> Union[DataRef, DataVal, Bracket, EOF]: - if METADATA in json_dict: - # Metadata Record - if Bracket.is_superclass(json_dict): - return Bracket.cons(json_dict) - else: - assert EOF.is_superclass(json_dict) - return EOF.cons(json_dict) - else: - # Data Record - if DataVal.is_superclass(json_dict): - return DataVal.cons(json_dict) - else: - assert DataRef.is_superclass(json_dict) - return DataRef.cons(json_dict) diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__init__.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__init__.py deleted file mode 100644 index 05fdd94..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .value import Value as DataVal -from .reference import Reference as DataRef - -__all__ = ["DataVal", "DataRef"] diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 11ae354..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/abstract.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/abstract.cpython-38.pyc deleted file mode 100644 index ba169d7..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/abstract.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/reference.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/reference.cpython-38.pyc deleted file mode 100644 index c683b6c..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/reference.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/value.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/value.cpython-38.pyc deleted file mode 100644 index 4134097..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/__pycache__/value.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/abstract.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/data/abstract.py deleted file mode 100644 index 31fc14b..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/abstract.py +++ /dev/null @@ -1,31 +0,0 @@ -from ..constants import * -from ..abstract import Entry -from ....logger import Future - -from abc import ABC, abstractmethod - - -class Data(Entry, Future, ABC): - def __init__(self, sk, gk, v): - Entry.__init__(self, sk, gk) - Future.__init__(self, v) - self.value = v - - def is_left(self): - return False - - def is_right(self): - return True - - def jsonify(self): - d = super().jsonify() - d[VAL] = self.value - return d - - @abstractmethod - def make_val(self): - ... - - @abstractmethod - def would_mat(self): - ... diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/reference.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/data/reference.py deleted file mode 100644 index 4476e9f..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/reference.py +++ /dev/null @@ -1,76 +0,0 @@ -from ..constants import * -from .abstract import Data -from ....logger.copy import deepcopy # type: ignore -from .... import shelf - -import cloudpickle -from pathlib import PurePath -from typing import Union -import json - - -class Reference(Data): - def __init__(self, sk, gk, v=None, r: Union[None, PurePath] = None): - assert bool(v is not None) != bool(r is not None) - super().__init__(sk, gk, v) - self.ref = r - self.val_saved = v is None and r is not None - - def make_val(self): - assert self.ref is not None - with open(self.ref, "rb") as f: - self.value = cloudpickle.load(f) - - def would_mat(self): - """ - For timing serialization costs - """ - cloudpickle.dumps(self.value) - - def jsonify(self): - assert self.ref is not None - assert ( - self.val_saved and self.ref.suffix == PKL_SFX - ), "Must call Reference.set_ref_and_dump(...) before jsonify()" - d = super().jsonify() - del d[VAL] - d[REF] = str(self.ref) - return d - - def set_ref(self, pkl_ref: PurePath): - self.ref = pkl_ref - - def dump(self): - assert ( - isinstance(self.ref, PurePath) and self.ref.suffix == PKL_SFX - ), "Must first set a reference path with a `.pkl` suffix" - with open(self.ref, "wb") as f: - cloudpickle.dump(self.value, f) - self.val_saved = True - self.value = None - - def set_ref_and_dump(self, pkl_ref: PurePath): - self.set_ref(pkl_ref) - self.dump() - - @staticmethod - def is_superclass(json_dict: dict): - assert bool(VAL in json_dict) != bool(REF in json_dict) - return REF in json_dict - - @classmethod - def cons(cls, json_dict: dict): - return cls( - json_dict[STATIC_KEY], json_dict[GLOBAL_KEY], v=None, r=json_dict[REF] - ) - - def promise(self): - self.promised = deepcopy(self.value) - self.value = self.promised - - def fulfill(self): - super().fulfill() - ref = shelf.get_pkl_ref() - assert ref is not None - self.set_ref_and_dump(ref) - return json.dumps(self.jsonify()) diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/value.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/data/value.py deleted file mode 100644 index be7e0a1..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/data/value.py +++ /dev/null @@ -1,35 +0,0 @@ -from ..constants import * -from .abstract import Data - -import json - - -class Value(Data): - def __init__(self, sk, gk, v): - super().__init__(sk, gk, v) - - def would_mat(self): - """ - For timing serialization costs - """ - d = self.jsonify() - json.dumps(d) - - def make_val(self): - pass - - @staticmethod - def is_superclass(json_dict: dict): - assert bool(VAL in json_dict) != bool(REF in json_dict) - return VAL in json_dict - - @classmethod - def cons(cls, json_dict: dict): - return cls(json_dict[STATIC_KEY], json_dict[GLOBAL_KEY], json_dict[VAL]) - - def promise(self): - self.promised = self.value - - def fulfill(self): - super().fulfill() - return json.dumps(self.jsonify()) diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__init__.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__init__.py deleted file mode 100644 index 984f684..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .bracket import Bracket -from .eof import EOF - -__all__ = ["Bracket", "EOF"] diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index bab9295..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/abstract.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/abstract.cpython-38.pyc deleted file mode 100644 index 59a2fe5..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/abstract.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/bracket.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/bracket.cpython-38.pyc deleted file mode 100644 index 5a5a071..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/bracket.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/eof.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/eof.cpython-38.pyc deleted file mode 100644 index 929558f..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/__pycache__/eof.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/abstract.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/abstract.py deleted file mode 100644 index f36231e..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/abstract.py +++ /dev/null @@ -1,23 +0,0 @@ -from ..constants import * -from ..abstract import Entry -from ....logger import Future - -from abc import ABC -import json - - -class Metadata(Entry, Future, ABC): - def __init__(self, sk, gk, meta): - Entry.__init__(self, sk, gk) - self.meta = meta - - def jsonify(self): - d = super().jsonify() - d[METADATA] = self.meta - return d - - def promise(self): - ... - - def fulfill(self): - return json.dumps(self.jsonify()) diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/bracket.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/bracket.py deleted file mode 100644 index 3ee2c90..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/bracket.py +++ /dev/null @@ -1,26 +0,0 @@ -from ..constants import * -from .abstract import Metadata - -VALID_BRACKETS = (LBRACKET, RBRACKET) - - -class Bracket(Metadata): - def __init__(self, sk, gk, meta, predicate=None, timestamp=None): - assert meta in VALID_BRACKETS - super().__init__(sk, gk, meta) - self.predicate = predicate - self.timestamp = timestamp - - def is_left(self): - return self.meta == LBRACKET - - def is_right(self): - return self.meta == RBRACKET - - @staticmethod - def is_superclass(json_dict: dict): - return METADATA in json_dict and json_dict[METADATA] in VALID_BRACKETS - - @classmethod - def cons(cls, json_dict: dict): - return cls(json_dict[STATIC_KEY], json_dict[GLOBAL_KEY], json_dict[METADATA]) diff --git a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/eof.py b/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/eof.py deleted file mode 100644 index 7cf1092..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/entry/metadata/eof.py +++ /dev/null @@ -1,36 +0,0 @@ -from ..constants import * -from .abstract import Metadata - -from typing import List - - -class EOF(Metadata): - def __init__(self, sparse: List[int], itc: int, commit_sha: str): - super().__init__(None, None, EOF_NAME) - self.sparse_checkpoints = sparse - self.iterations_count = itc - self.commit_sha = commit_sha - - def is_left(self): - return False - - def is_right(self): - return False - - def jsonify(self): - d = dict() - d[METADATA] = EOF_NAME - d[SPARSE_CHECKPOINTS] = self.sparse_checkpoints - d[ITERATIONS_COUNT] = int(self.iterations_count) - d["COMMIT_SHA"] = self.commit_sha - return d - - @staticmethod - def is_superclass(json_dict: dict): - return METADATA in json_dict and json_dict[METADATA] == EOF_NAME - - @classmethod - def cons(cls, json_dict: dict): - sparse = json_dict[SPARSE_CHECKPOINTS] - assert isinstance(sparse, List) - return cls(sparse, int(json_dict[ITERATIONS_COUNT]), json_dict["COMMIT_SHA"]) diff --git a/.venv/lib/python3.8/site-packages/flor/journal/file.py b/.venv/lib/python3.8/site-packages/flor/journal/file.py deleted file mode 100644 index 4dcf743..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/file.py +++ /dev/null @@ -1,20 +0,0 @@ -from .entry import DataVal, DataRef, Bracket, EOF, make_entry -from flor import shelf - -import json -from typing import Union, List - - -def read() -> List[Union[DataRef, DataVal, Bracket, EOF]]: - entries: List[Union[DataRef, DataVal, Bracket, EOF]] = [] - index = shelf.get_index() - if index is not None: - with open(index, "r") as f: - for line in f: - log_record = make_entry(json.loads(line.strip())) - entries.append(log_record) - return entries - raise RuntimeError("Shelf not initialized. Did you call shelf.mk_job?") - - -__all__ = ["read"] diff --git a/.venv/lib/python3.8/site-packages/flor/journal/journal.py b/.venv/lib/python3.8/site-packages/flor/journal/journal.py deleted file mode 100644 index df1aa6b..0000000 --- a/.venv/lib/python3.8/site-packages/flor/journal/journal.py +++ /dev/null @@ -1,67 +0,0 @@ -from typing import List, Union - -from flor import flags -from flor.constants import * -from flor.journal.entry import EOF -from flor.tree import Tree -from flor.tree.window import Capsule - -from . import file - - -class Journal: - def __init__(self): - self.tree = Tree() - self.sub_tree = None - self.entries = None - - def read(self): - self.entries = file.read() - self.tree.parse(self.entries) - - def get_segment_window(self) -> List[Capsule]: - assert flags.PID.ngpus <= self.tree.iterations_count - if self.tree.sparse_checkpoints: - assert ( - flags.PID.ngpus <= len(self.tree.sparse_checkpoints) + 1 - ), f"Not enough checkpoints. Max degree of parallelism: {len(self.tree.sparse_checkpoints) + 1}" - if flags.MODE is REPLAY_MODE.weak and flags.PID.pid > 1: - self._advance_head() - assert self.sub_tree is not None - return self.sub_tree.get_segment() - return self.tree.get_segment() - - def as_tree(self) -> Tree: - if not flags.REPLAY or flags.MODE is REPLAY_MODE.strong or flags.PID.pid == 1: - return self.tree - else: - assert self.sub_tree is not None - return self.sub_tree - - def get_eof(self, commit_sha: str): - tree = self.as_tree() - return EOF(tree.sparse_checkpoints, tree.iterations_count, commit_sha) - - def _advance_head(self): - """ - Used for checkpoint resume, - ignores journal entries that precede the first epoch of work - """ - assert self.sub_tree is None, "Need a fresh Tree to feed" - self.sub_tree = Tree() - epoch_to_init: Union[int, None] = self.tree.get_resume_epoch() - if epoch_to_init is not None: - assert self.tree.root is not None - target = self.tree[self.tree.root.static_key].blocks[epoch_to_init] - feeding = False - assert self.entries is not None - for journal_entry in self.entries: - if ( - not feeding - and journal_entry.is_left() - and journal_entry.sk == target.static_key - and journal_entry.gk == target.global_key - ): - feeding = True - if feeding: - self.sub_tree.feed_entry(journal_entry) diff --git a/.venv/lib/python3.8/site-packages/flor/logger/__init__.py b/.venv/lib/python3.8/site-packages/flor/logger/__init__.py deleted file mode 100644 index b26f551..0000000 --- a/.venv/lib/python3.8/site-packages/flor/logger/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .logger import Logger -from .future import Future diff --git a/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 93ee300..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/copy.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/copy.cpython-38.pyc deleted file mode 100644 index 73744d3..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/copy.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/future.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/future.cpython-38.pyc deleted file mode 100644 index b373d4b..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/future.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/logger.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/logger.cpython-38.pyc deleted file mode 100644 index be9c9f0..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/logger/__pycache__/logger.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/logger/copy.py b/.venv/lib/python3.8/site-packages/flor/logger/copy.py deleted file mode 100644 index d75d55b..0000000 --- a/.venv/lib/python3.8/site-packages/flor/logger/copy.py +++ /dev/null @@ -1,353 +0,0 @@ -# type: ignore -"""Generic (shallow and deep) copying operations. - -Interface summary: - - import copy - - x = copy.copy(y) # make a shallow copy of y - x = copy.deepcopy(y) # make a deep copy of y - -For module specific errors, copy.Error is raised. - -The difference between shallow and deep copying is only relevant for -compound objects (objects that contain other objects, like lists or -class instances). - -- A shallow copy constructs a new compound object and then (to the - extent possible) inserts *the same objects* into it that the - original contains. - -- A deep copy constructs a new compound object and then, recursively, - inserts *copies* into it of the objects found in the original. - -Two problems often exist with deep copy operations that don't exist -with shallow copy operations: - - a) recursive objects (compound objects that, directly or indirectly, - contain a reference to themselves) may cause a recursive loop - - b) because deep copy copies *everything* it may copy too much, e.g. - administrative data structures that should be shared even between - copies - -Python's deep copy operation avoids these problems by: - - a) keeping a table of objects already copied during the current - copying pass - - b) letting user-defined classes override the copying operation or the - set of components copied - -This version does not copy types like module, class, function, method, -nor stack trace, stack frame, nor file, socket, window, nor array, nor -any similar types. - -Classes can use the same interfaces to control copying that they use -to control pickling: they can define methods called __getinitargs__(), -__getstate__() and __setstate__(). See the documentation for module -"pickle" for information on these methods. -""" - -import types -import weakref -from copyreg import dispatch_table - - -class Error(Exception): - pass - - -error = Error # backward compatibility - -try: - from org.python.core import PyStringMap -except ImportError: - PyStringMap = None - -__all__ = ["Error", "copy", "deepcopy"] - - -def copy(x): - """Shallow copy operation on arbitrary Python objects. - - See the module's __doc__ string for more info. - """ - - cls = type(x) - - copier = _copy_dispatch.get(cls) - if copier: - return copier(x) - - if issubclass(cls, type): - # treat it as a regular class: - return _copy_immutable(x) - - copier = getattr(cls, "__copy__", None) - if copier is not None: - return copier(x) - - reductor = dispatch_table.get(cls) - if reductor is not None: - rv = reductor(x) - else: - reductor = getattr(x, "__reduce_ex__", None) - if reductor is not None: - rv = reductor(4) - else: - reductor = getattr(x, "__reduce__", None) - if reductor: - rv = reductor() - else: - raise Error("un(shallow)copyable object of type %s" % cls) - - if isinstance(rv, str): - return x - return _reconstruct(x, None, *rv) - - -_copy_dispatch = d = {} - - -def _copy_immutable(x): - return x - - -for t in ( - type(None), - int, - float, - bool, - complex, - str, - tuple, - bytes, - frozenset, - type, - range, - slice, - property, - types.BuiltinFunctionType, - type(Ellipsis), - type(NotImplemented), - types.FunctionType, - weakref.ref, -): - d[t] = _copy_immutable -t = getattr(types, "CodeType", None) -if t is not None: - d[t] = _copy_immutable - -d[list] = list.copy -d[dict] = dict.copy -d[set] = set.copy -d[bytearray] = bytearray.copy - -if PyStringMap is not None: - d[PyStringMap] = PyStringMap.copy - -del d, t - - -def deepcopy(x, memo=None, _nil=[]): - """Deep copy operation on arbitrary Python objects. - - See the module's __doc__ string for more info. - """ - - if memo is None: - memo = {} - - d = id(x) - - y = memo.get(d, _nil) - # y is the unique identifier (address) of x IF X in memo; else it is NILL - if y is not _nil: - # x has already been copied and is stored in the memo - return y - - cls = type(x) - - copier = _deepcopy_dispatch.get(cls) - if copier is not None: - y = copier(x, memo) - else: - if issubclass(cls, type): - y = _deepcopy_atomic(x, memo) - elif str(cls) == "" and "cpu" not in x.device.type: - # Using a string so we don't have to import torch - # We have to ensure tensor is not already in CPU -- else .cpu() is no-op - y = x.cpu() # This achieves a fast copy - else: - copier = getattr(x, "__deepcopy__", None) - if copier is not None: - y = copier(memo) - else: - reductor = dispatch_table.get(cls) - if reductor: - rv = reductor(x) - else: - reductor = getattr(x, "__reduce_ex__", None) - if reductor is not None: - rv = reductor(4) - else: - reductor = getattr(x, "__reduce__", None) - if reductor: - rv = reductor() - else: - raise Error("un(deep)copyable object of type %s" % cls) - if isinstance(rv, str): - y = x - else: - y = _reconstruct(x, memo, *rv) - - # If is its own copy, don't memoize. - if y is not x: - memo[d] = y - _keep_alive(x, memo) # Make sure x lives at least as long as d - return y - - -_deepcopy_dispatch = d = {} - - -def _deepcopy_atomic(x, memo): - return x - - -d[type(None)] = _deepcopy_atomic -d[type(Ellipsis)] = _deepcopy_atomic -d[type(NotImplemented)] = _deepcopy_atomic -d[int] = _deepcopy_atomic -d[float] = _deepcopy_atomic -d[bool] = _deepcopy_atomic -d[complex] = _deepcopy_atomic -d[bytes] = _deepcopy_atomic -d[str] = _deepcopy_atomic -d[types.CodeType] = _deepcopy_atomic -d[type] = _deepcopy_atomic -d[types.BuiltinFunctionType] = _deepcopy_atomic -d[types.FunctionType] = _deepcopy_atomic -d[weakref.ref] = _deepcopy_atomic -d[property] = _deepcopy_atomic - - -def _deepcopy_list(x, memo, deepcopy=deepcopy): - y = [] - memo[id(x)] = y - append = y.append - for a in x: - append(deepcopy(a, memo)) - return y - - -d[list] = _deepcopy_list - - -def _deepcopy_tuple(x, memo, deepcopy=deepcopy): - y = [deepcopy(a, memo) for a in x] - # We're not going to put the tuple in the memo, but it's still important we - # check for it, in case the tuple contains recursive mutable structures. - try: - return memo[id(x)] - except KeyError: - pass - for k, j in zip(x, y): - if k is not j: - y = tuple(y) - break - else: - y = x - return y - - -d[tuple] = _deepcopy_tuple - - -def _deepcopy_dict(x, memo, deepcopy=deepcopy): - y = {} - memo[id(x)] = y - for key, value in x.items(): - y[deepcopy(key, memo)] = deepcopy(value, memo) - return y - - -d[dict] = _deepcopy_dict -if PyStringMap is not None: - d[PyStringMap] = _deepcopy_dict - - -def _deepcopy_method(x, memo): # Copy instance methods - return type(x)(x.__func__, deepcopy(x.__self__, memo)) - - -d[types.MethodType] = _deepcopy_method - -del d - - -def _keep_alive(x, memo): - """Keeps a reference to the object x in the memo. - - Because we remember objects by their id, we have - to assure that possibly temporary objects are kept - alive by referencing them. - We store a reference at the id of the memo, which should - normally not be used unless someone tries to deepcopy - the memo itself... - """ - try: - memo[id(memo)].append(x) - except KeyError: - # aha, this is the first one :-) - memo[id(memo)] = [x] - - -def _reconstruct( - x, memo, func, args, state=None, listiter=None, dictiter=None, deepcopy=deepcopy -): - deep = memo is not None - if deep and args: - args = (deepcopy(arg, memo) for arg in args) - y = func(*args) - if deep: - memo[id(x)] = y - - if state is not None: - if deep: - state = deepcopy(state, memo) - if hasattr(y, "__setstate__"): - y.__setstate__(state) - else: - if isinstance(state, tuple) and len(state) == 2: - state, slotstate = state - else: - slotstate = None - if state is not None: - y.__dict__.update(state) - if slotstate is not None: - for key, value in slotstate.items(): - setattr(y, key, value) - - if listiter is not None: - if deep: - for item in listiter: - item = deepcopy(item, memo) - y.append(item) - else: - for item in listiter: - y.append(item) - if dictiter is not None: - if deep: - for key, value in dictiter: - key = deepcopy(key, memo) - value = deepcopy(value, memo) - y[key] = value - else: - for key, value in dictiter: - y[key] = value - return y - - -del types, weakref, PyStringMap diff --git a/.venv/lib/python3.8/site-packages/flor/logger/future.py b/.venv/lib/python3.8/site-packages/flor/logger/future.py deleted file mode 100644 index 6bd110f..0000000 --- a/.venv/lib/python3.8/site-packages/flor/logger/future.py +++ /dev/null @@ -1,24 +0,0 @@ -from abc import ABC, abstractmethod - - -class Future(ABC): - def __init__(self, v): - self.value = v - self.promised = None - - @abstractmethod - def promise(self): - """ - Synchronous move or copy for data protection - Example: - self.promised = deepcopy(self.value) - """ - ... - - @abstractmethod - def fulfill(self) -> str: - """ - Finishes intended work on stored version of self.value - """ - assert self.promised is not None - return "" diff --git a/.venv/lib/python3.8/site-packages/flor/logger/logger.py b/.venv/lib/python3.8/site-packages/flor/logger/logger.py deleted file mode 100644 index 2f84720..0000000 --- a/.venv/lib/python3.8/site-packages/flor/logger/logger.py +++ /dev/null @@ -1,90 +0,0 @@ -import glob -import os -from pathlib import Path -from typing import List, Generator - -from .. import shelf -from .future import Future - - -class Logger: - def __init__(self, path=None, buf_size=None): - self.path = Path(path) if path is not None else None - self.buffer = Buffer() if buf_size is None else Buffer(buf_size) - self.flush_count = 0 - - def set_path(self, path): - self.path = Path(path) - - def append(self, o: Future): - assert self.path is not None, "Logger path not set." - self.buffer.append(o) - if self.buffer.is_full(): - self.flush() - - def close(self): - if len(self.buffer) > 0: - self.flush(is_final=True) - latest = shelf.get_latest() - assert latest is not None - assert self.path is not None - if latest.exists(): - latest.unlink() - latest.symlink_to(self.path) - # TODO: spool - - def flush(self, is_final=False): - assert self.path is not None - self.flush_count += 1 - pid = os.fork() - if not pid: - self._flush_buffer() - if is_final: - p = self.path.with_name(self.path.stem + "_*" + self.path.suffix) - with open(self.path, "wb") as out_f: - for pi in glob.glob(p.as_posix()): - with open(pi, "rb") as in_f: - out_f.write(in_f.read()) - os.remove(pi) - else: - self.buffer.clear() - - def force(self): - self._flush_buffer() - - def _flush_buffer(self): - assert isinstance(self.path, Path) - p = self.path.with_name( - self.path.stem + f"_{self.flush_count}" + self.path.suffix - ) - with open(p, "w") as f: - for o in self.buffer.flush(): - assert o is not None - f.write(o + os.linesep) - - -class Buffer: - def __init__(self, size=1024): - self._b: List[Future] = [] - self.size = size - - def append(self, o: Future): - assert isinstance( - o, Future - ), "object passed to flor Logger must implement Future interface" - o.promise() - self._b.append(o) - - def is_full(self): - return len(self._b) >= self.size - - def clear(self): - self._b[:] = [] - - def flush(self) -> Generator[str, None, None]: - for o in self._b: - yield o.fulfill() - self.clear() - - def __len__(self): - return len(self._b) diff --git a/.venv/lib/python3.8/site-packages/flor/pin.py b/.venv/lib/python3.8/site-packages/flor/pin.py deleted file mode 100644 index 1a75abc..0000000 --- a/.venv/lib/python3.8/site-packages/flor/pin.py +++ /dev/null @@ -1,22 +0,0 @@ -from . import flags -from typing import Any, Dict, TypeVar, Union - -LITERALS = Union[int, float, bool, str] -T = TypeVar("T", int, float, bool, str) -kvs: Dict[str, LITERALS] = dict() - - -def pin(name: str, value: T) -> T: - """ - Helper method for pinning random number generator seeds - """ - if flags.NAME is None: - return value - if flags.REPLAY: - assert name in kvs - assert type(value) == type(kvs[name]) - return kvs[name] # type: ignore - else: - assert name not in kvs - kvs[name] = value - return value diff --git a/.venv/lib/python3.8/site-packages/flor/shelf/__init__.py b/.venv/lib/python3.8/site-packages/flor/shelf/__init__.py deleted file mode 100644 index 6748a9c..0000000 --- a/.venv/lib/python3.8/site-packages/flor/shelf/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .directory import mk_job, get_index, get_latest, get_pkl_ref, verify diff --git a/.venv/lib/python3.8/site-packages/flor/shelf/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/shelf/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index b6b9cd5..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/shelf/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/shelf/__pycache__/directory.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/shelf/__pycache__/directory.cpython-38.pyc deleted file mode 100644 index 768a763..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/shelf/__pycache__/directory.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/shelf/directory.py b/.venv/lib/python3.8/site-packages/flor/shelf/directory.py deleted file mode 100644 index fae3395..0000000 --- a/.venv/lib/python3.8/site-packages/flor/shelf/directory.py +++ /dev/null @@ -1,54 +0,0 @@ -import uuid -from datetime import datetime -from pathlib import Path, PurePath -from typing import Optional, Union - -from flor import flags - -home: Path = Path.home() - -florin: Path = home / ".flor" -florin.mkdir(exist_ok=True) - -job: Optional[Path] = None -data: Optional[Path] = None -timestamp: Optional[str] = None - - -def mk_job(name: str): - global timestamp, job, data - assert isinstance(name, str) - timestamp = datetime.now().isoformat() - job = florin / name - job.mkdir(exist_ok=True) - data = job / "data" - data.mkdir(exist_ok=True) - - -def get_index() -> Optional[Path]: - if job is not None and timestamp is not None: - if flags.REPLAY: - assert flags.INDEX is not None - return job / flags.INDEX.with_suffix(".json") - else: - return job / PurePath(timestamp).with_suffix(".json") - else: - return None - - -def get_latest() -> Optional[Path]: - return job / PurePath("latest").with_suffix(".json") if job is not None else None - - -def get_pkl_ref() -> Optional[Path]: - return ( - data / PurePath(uuid.uuid4().hex).with_suffix(".pkl") - if data is not None - else None - ) - - -def verify(path: Union[PurePath, str]) -> bool: - assert flags.NAME is not None - resolved_path = florin / flags.NAME / path - return resolved_path.exists() diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/__init__.py b/.venv/lib/python3.8/site-packages/flor/skipblock/__init__.py deleted file mode 100644 index 505b679..0000000 --- a/.venv/lib/python3.8/site-packages/flor/skipblock/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .abstract import SkipBlock diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 4f2e3de..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/abstract.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/abstract.cpython-38.pyc deleted file mode 100644 index 2a6a8e2..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/abstract.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/readblock.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/readblock.cpython-38.pyc deleted file mode 100644 index e53d281..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/readblock.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/seemblock.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/seemblock.cpython-38.pyc deleted file mode 100644 index 244320e..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/seemblock.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/writeblock.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/writeblock.cpython-38.pyc deleted file mode 100644 index 33194ec..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/skipblock/__pycache__/writeblock.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/abstract.py b/.venv/lib/python3.8/site-packages/flor/skipblock/abstract.py deleted file mode 100644 index 4995ac8..0000000 --- a/.venv/lib/python3.8/site-packages/flor/skipblock/abstract.py +++ /dev/null @@ -1,24 +0,0 @@ -from .seemblock import SeemBlock -from .readblock import ReadBlock -from .writeblock import WriteBlock - -from flor import flags - - -class SkipBlock(SeemBlock): - @staticmethod - def step_into(block_name: str, probed=None): - if flags.NAME is not None: - raise RuntimeError("SkipBlock missing dynamic linking") - return True - - @staticmethod - def end(*args, values=None): - if flags.NAME is not None: - raise RuntimeError("SkipBlock missing dynamic linking") - - @staticmethod - def bind(): - block = ReadBlock if flags.REPLAY else WriteBlock - SkipBlock.step_into = block.step_into # type: ignore - SkipBlock.end = block.end diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/readblock.py b/.venv/lib/python3.8/site-packages/flor/skipblock/readblock.py deleted file mode 100644 index d2b5e81..0000000 --- a/.venv/lib/python3.8/site-packages/flor/skipblock/readblock.py +++ /dev/null @@ -1,56 +0,0 @@ -from .seemblock import SeemBlock - -from flor import flags -from flor.journal.entry import Bracket, LBRACKET - -from typing import List, Dict - - -class ReadBlock(SeemBlock): - dynamic_identifiers: Dict[str, int] = dict() - pda: List[Bracket] = [] - - @staticmethod - def step_into(block_name: str, probed=None): - if probed is None: - probed = flags.PID.ngpus > 1 - assert isinstance(block_name, str) - dynamic_id = ReadBlock.dynamic_identifiers.get(block_name, 0) - ReadBlock.dynamic_identifiers[block_name] = dynamic_id + 1 - - lbracket = Bracket( - block_name, dynamic_id, LBRACKET, predicate=not flags.RESUMING and probed - ) - ReadBlock.pda.append(lbracket) - return lbracket.predicate - - @staticmethod - def end(*args, values=None): - lbracket = ReadBlock.pda.pop() - block = ReadBlock.journal.as_tree()[lbracket.sk].blocks[lbracket.gk] - if not lbracket.predicate: - for data_record, arg in zip(block.data_records, args): - data_record.make_val() - value_serialized = data_record.value - if hasattr(arg, "load_state_dict"): - # PyTorch support - arg.load_state_dict(value_serialized) - elif isinstance(arg, list): - assert isinstance(value_serialized, list) - arg[:] = value_serialized - elif isinstance(arg, dict): - assert isinstance(value_serialized, dict) - arg.update(value_serialized) - elif hasattr(arg, "__dict__"): - if isinstance(value_serialized, dict): - arg.__dict__.update(arg) - else: - assert type(arg) == type(value_serialized) - arg.__dict__.update(value_serialized.__dict__) - else: - # TODO: ... - raise RuntimeError( - "TODO: add hooks for user-defined de-serialization" - ) - # TODO: ... - assert values is None, "TODO: Add support for literals/atomics" diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/seemblock.py b/.venv/lib/python3.8/site-packages/flor/skipblock/seemblock.py deleted file mode 100644 index a2b278a..0000000 --- a/.venv/lib/python3.8/site-packages/flor/skipblock/seemblock.py +++ /dev/null @@ -1,19 +0,0 @@ -from ..journal import Journal -from ..logger import Logger - -from abc import ABC, abstractmethod - - -class SeemBlock(ABC): - journal = Journal() - logger = Logger() - - @staticmethod - @abstractmethod - def step_into(block_name: str, probed=None): - ... - - @staticmethod - @abstractmethod - def end(*args, values=None): - ... diff --git a/.venv/lib/python3.8/site-packages/flor/skipblock/writeblock.py b/.venv/lib/python3.8/site-packages/flor/skipblock/writeblock.py deleted file mode 100644 index 7fe7189..0000000 --- a/.venv/lib/python3.8/site-packages/flor/skipblock/writeblock.py +++ /dev/null @@ -1,101 +0,0 @@ -from .seemblock import SeemBlock - -from flor import flags -from flor import shelf -from flor.journal.entry import DataRef, DataVal, Bracket, LBRACKET, RBRACKET - -import time -from typing import Dict, List, Union - - -class WriteBlock(SeemBlock): - scaling_factor = 1.38 - dynamic_identifiers: Dict[str, int] = dict() - pda: List[Bracket] = [] - - @staticmethod - def step_into(block_name: str, probed=None): - assert isinstance(block_name, str) - dynamic_id = WriteBlock.dynamic_identifiers.get(block_name, 0) - WriteBlock.dynamic_identifiers[block_name] = dynamic_id + 1 - - lbracket = Bracket( - block_name, dynamic_id, LBRACKET, predicate=True, timestamp=time.time() - ) - WriteBlock.journal.as_tree().feed_entry(lbracket) - - WriteBlock.logger.append(lbracket) - WriteBlock.pda.append(lbracket) - return lbracket.predicate - - @staticmethod - def end(*args, values=None): - lbracket = WriteBlock.pda.pop() - assert lbracket.timestamp is not None - block_group = WriteBlock.journal.as_tree()[lbracket.sk] - block = block_group.peek_block() - assert block.global_key == lbracket.gk - block_group.tick_execution(time.time() - lbracket.timestamp) - if not args: - rbracket = Bracket(lbracket.sk, lbracket.gk, RBRACKET) - WriteBlock.journal.as_tree().feed_entry(rbracket) - WriteBlock.logger.append(rbracket) - block_group.set_mat_time(0) - else: - data_records = [] - - start_time = time.time() - for arg in args: - data_record = val_to_record(arg, lbracket) - data_records.append(data_record) - block_group.should_time_mat() and data_record.would_mat() - block_group.set_mat_time(time.time() - start_time) - - if WriteBlock._should_materialize(block_group): - for data_record in data_records: - WriteBlock.journal.as_tree().feed_entry(data_record) - WriteBlock.logger.append(data_record) - block_group.tick_materialization() - else: - rbracket = Bracket(lbracket.sk, lbracket.gk, RBRACKET) - WriteBlock.journal.as_tree().feed_entry(rbracket) - WriteBlock.logger.append(rbracket) - - @staticmethod - def _should_materialize(block_group): - assert block_group.materialization_time is not None - assert block_group.computation_time is not None - - block = block_group.peek_block() - - # Must align successor checkpoints for periodic checkpointing - if block.force_mat: - return True - - # First consider atomic case (always/never) - ratio = block_group.materialization_time / block_group.computation_time - threshold = min(1 / (1 + WriteBlock.scaling_factor), flags.EPSILON) - if ratio < threshold: - return True - - # Then account for parallelism speedup - if block.parent is None: - threshold *= block_group.executions_count / ( - block_group.materializations_count + 1 - ) - if ratio < threshold: - WriteBlock.journal.as_tree().add_sparse_checkpoint() - block.force_mat_successors() - return True - - return False - - -def val_to_record(arg, lbracket: Bracket) -> Union[DataRef, DataVal]: - if hasattr(arg, "state_dict"): - arg = getattr(arg, "state_dict")() - - if type(arg) in [type(None), int, float, bool, str]: - return DataVal(lbracket.sk, lbracket.gk, arg) - else: - return DataRef(lbracket.sk, lbracket.gk, arg) diff --git a/.venv/lib/python3.8/site-packages/flor/tree/__init__.py b/.venv/lib/python3.8/site-packages/flor/tree/__init__.py deleted file mode 100644 index 498ee1a..0000000 --- a/.venv/lib/python3.8/site-packages/flor/tree/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .tree import Tree diff --git a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index ed71f9f..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/block.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/block.cpython-38.pyc deleted file mode 100644 index 4d9679a..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/block.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/group.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/group.cpython-38.pyc deleted file mode 100644 index 31774aa..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/group.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/tree.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/tree.cpython-38.pyc deleted file mode 100644 index 9ed8404..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/tree.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/window.cpython-38.pyc b/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/window.cpython-38.pyc deleted file mode 100644 index 06040a1..0000000 Binary files a/.venv/lib/python3.8/site-packages/flor/tree/__pycache__/window.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/flor/tree/block.py b/.venv/lib/python3.8/site-packages/flor/tree/block.py deleted file mode 100644 index 38c2556..0000000 --- a/.venv/lib/python3.8/site-packages/flor/tree/block.py +++ /dev/null @@ -1,36 +0,0 @@ -from flor.journal.entry import DataVal, DataRef, Bracket -from typing import Union, List - - -class Block: - def __init__(self, log_record: Bracket, parent=None): - assert log_record.is_left() - - self.child: Union["Block", None] = None - self.parent: Union["Block", None] = parent - self.successor: Union["Block", None] = None - - self.static_key = log_record.sk - self.global_key = log_record.gk - self.data_records: List[Union[DataVal, DataRef]] = [] - self.right_fed = False - self.force_mat = False - - def belongs_in_block(self, data_record: Union[DataVal, DataRef, Bracket]): - assert data_record.is_right() - return data_record.sk == self.static_key and data_record.gk == self.global_key - - def feed_entry(self, data_record: Union[DataVal, DataRef, Bracket]): - assert self.belongs_in_block(data_record) - self.right_fed = True - if not isinstance(data_record, Bracket): - # If it's not a RBRACKET then it's a data record - self.data_records.append(data_record) - - def force_mat_successors(self): - assert self.parent is None - first_static_key = self.static_key - block = self.successor - while block is not None and block.static_key != first_static_key: - block.force_mat = True - block = block.successor diff --git a/.venv/lib/python3.8/site-packages/flor/tree/group.py b/.venv/lib/python3.8/site-packages/flor/tree/group.py deleted file mode 100644 index 44a234b..0000000 --- a/.venv/lib/python3.8/site-packages/flor/tree/group.py +++ /dev/null @@ -1,36 +0,0 @@ -from .block import Block -from typing import List - - -class BlockGroup: - def __init__(self, first: Block): - self.blocks: List[Block] = [ - first, - ] - - self.materialization_time = None - self.computation_time = None - self.executions_count = 0 - self.materializations_count = 0 - - def tick_execution(self, t): - self.executions_count += 1 - if self.computation_time is None: - self.computation_time = t - - def should_time_mat(self): - assert self.executions_count > 0 - return self.executions_count == 1 - - def tick_materialization(self): - self.materializations_count += 1 - - def set_mat_time(self, t): - if self.materialization_time is None: - self.materialization_time = t - - def add_block(self, block: Block): - self.blocks.append(block) - - def peek_block(self) -> Block: - return self.blocks[-1] diff --git a/.venv/lib/python3.8/site-packages/flor/tree/tree.py b/.venv/lib/python3.8/site-packages/flor/tree/tree.py deleted file mode 100644 index a4f74bf..0000000 --- a/.venv/lib/python3.8/site-packages/flor/tree/tree.py +++ /dev/null @@ -1,96 +0,0 @@ -from .block import Block -from .group import BlockGroup -from .window import Window -from flor.journal.entry import DataVal, DataRef, Bracket, EOF - -from collections import OrderedDict as ODict -from typing import List, Union, Dict - - -class Tree: - def __init__(self, log_entry: Bracket = None): - self.hash: Dict[str, BlockGroup] = ODict() - self.block = None - - if log_entry is not None: - assert log_entry.is_left() - self.block = Block(log_entry) - self._hash(self.block) - - self.root = self.block - - self.sparse_checkpoints: List[int] = [] - self.iterations_count = 0 - - def __getitem__(self, item) -> BlockGroup: - return self.hash[item] - - def _hash(self, block: Block): - if block.static_key in self.hash: - self.hash[block.static_key].add_block(block) - else: - self.hash[block.static_key] = BlockGroup(block) - - def add_sparse_checkpoint(self): - # print(f"SPARSE CHECKPOINT at {self.iterations_count - 1}") - self.sparse_checkpoints.append(self.iterations_count - 1) - - def feed_entry(self, log_entry: Union[DataRef, DataVal, Bracket, EOF]): - if self.root is None: - assert self.block is None - assert log_entry.is_left() - assert isinstance(log_entry, Bracket) - self.block = Block(log_entry) - self.root = self.block - self._hash(self.block) - elif log_entry.is_left(): - assert self.block is not None - if self.block.right_fed: - assert isinstance(log_entry, Bracket) - successor = Block(log_entry, self.block.parent) - self.block.successor = successor - self.block = successor - self._hash(successor) - else: - assert isinstance(log_entry, Bracket) - child = Block(log_entry, self.block) - self.block.child = child - self.block = child - self._hash(child) - elif log_entry.is_right(): - assert self.block is not None - assert not isinstance(log_entry, EOF) - if self.block.belongs_in_block(log_entry): - self.block.feed_entry(log_entry) - else: - assert ( - self.block.parent is not None - and self.block.parent.belongs_in_block(log_entry) - ) - self.block = self.block.parent - self.block.feed_entry(log_entry) - else: - assert isinstance(log_entry, EOF) - self.sparse_checkpoints = log_entry.sparse_checkpoints - self.iterations_count = log_entry.iterations_count - - if log_entry.is_left() and log_entry.sk == self.root.static_key: - self.iterations_count += 1 - - def parse(self, records: List[Union[DataRef, DataVal, Bracket, EOF]]): - assert self.block is None, "Need a new Tree for parsing" - for rec in records: - self.feed_entry(rec) - - def get_resume_epoch(self): - """ - Gets the predecessor epoch for weak initialization, - if such epoch exists, else None. - """ - return Window(self.iterations_count, self.sparse_checkpoints).get_resume_epoch() - - def get_segment(self): - """ - Gets the relevant work segment, including tagged initialization epochs - """ - return Window(self.iterations_count, self.sparse_checkpoints).get_work_segment() diff --git a/.venv/lib/python3.8/site-packages/flor/tree/window.py b/.venv/lib/python3.8/site-packages/flor/tree/window.py deleted file mode 100644 index 64637b6..0000000 --- a/.venv/lib/python3.8/site-packages/flor/tree/window.py +++ /dev/null @@ -1,112 +0,0 @@ -from flor import flags -from typing import List, Union, Optional -from ..constants import * - -NO_INIT = None - - -class Window: - def __init__(self, iterations_count: int, sparse_checkpoints: List[int]): - self.iterations_count = iterations_count - self.extended_sparse_checkpoints: List[Optional[int]] = list(sparse_checkpoints) - self.extended_sparse_checkpoints.insert(0, NO_INIT) - - def get_work_segment(self): - if self._is_sparse(): - # Only a subset of epochs are valid entries - # Only weak initialization is possible - assert flags.MODE is REPLAY_MODE.weak - lo = self._sparse() - hi = self._sparse(hi=True) - assert hi is not None - return [ - Capsule(True, lo), - ] + [Capsule(False, e) for e in range(lo + 1 if lo is not None else 0, hi)] - else: - # All epochs are valid entries - our_segment = self._get_segment_helper(self.iterations_count)[ - flags.PID.pid - 1 - ] - assert ( - our_segment - ), "TODO: Handle case when user allocs more partitions than there is work." - if flags.MODE is REPLAY_MODE.weak: - # Asks to initialize predecessor - return [ - Capsule(True, self._dense()), - ] + [Capsule(False, e) for e in our_segment] - else: - idx = our_segment[0] - assert idx is not None - return [Capsule(True, e) for e in range(idx)] + [ - Capsule(False, e) for e in our_segment - ] - - def get_resume_epoch(self): - """ - i : the epoch from which to resume (weak initialization) - """ - if flags.MODE is REPLAY_MODE.strong: - return None - if self._is_sparse(): - return self._sparse() - else: - return self._dense() - - def _is_sparse(self): - is_sparse = len(self.extended_sparse_checkpoints) > 1 - assert ( - is_sparse - or len(self.extended_sparse_checkpoints) == 1 - and self.extended_sparse_checkpoints[0] is NO_INIT - ) - return is_sparse - - def _get_segment_helper(self, num_resume_points): - segments: List[List[Optional[int]]] = [[] for _ in range(flags.PID.ngpus)] - for pt in range(num_resume_points): - segments[pt % flags.PID.ngpus].append(None) - i = 0 - for j in range(flags.PID.ngpus): - for k in range(len(segments[j])): - segments[j][k] = i - i += 1 - assert i == num_resume_points - return segments - - def _sparse(self, hi=False) -> Optional[int]: - our_segment = self._get_segment_helper(len(self.extended_sparse_checkpoints))[ - flags.PID.pid - 1 - ] - # TODO: ... - assert ( - our_segment - ), "TODO: Handle case when user allocs more partitions than there is work." - # Re-index the segment with respect to self.sparse_checkpoints - if not hi: - assert our_segment[0] is not None - return self.extended_sparse_checkpoints[our_segment[0]] - else: - if flags.PID.pid == flags.PID.ngpus: - # This is the last segment - return self.iterations_count - else: - # There exists a greater segment - idx = our_segment[-1] - assert idx is not None - return self.extended_sparse_checkpoints[idx + 1] - - def _dense(self) -> Optional[int]: - our_segment = self._get_segment_helper(self.iterations_count)[flags.PID.pid - 1] - # TODO: ... - assert ( - our_segment - ), "TODO: Handle case when user allocs more partitions than there is work." - pred_epoch = our_segment[0] - 1 if our_segment[0] else NO_INIT - return pred_epoch - - -class Capsule: - def __init__(self, init_only: bool, epoch: Optional[int]): - self.init_only = init_only - self.epoch = epoch diff --git a/.venv/lib/python3.8/site-packages/git/__init__.py b/.venv/lib/python3.8/site-packages/git/__init__.py deleted file mode 100644 index a39f540..0000000 --- a/.venv/lib/python3.8/site-packages/git/__init__.py +++ /dev/null @@ -1,90 +0,0 @@ -# __init__.py -# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors -# -# This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php -# flake8: noqa -#@PydevCodeAnalysisIgnore -from git.exc import * # @NoMove @IgnorePep8 -import inspect -import os -import sys -import os.path as osp - -from typing import Optional -from git.types import PathLike - -__version__ = '3.1.26' - - -#{ Initialization -def _init_externals() -> None: - """Initialize external projects by putting them into the path""" - if __version__ == '3.1.26' and 'PYOXIDIZER' not in os.environ: - sys.path.insert(1, osp.join(osp.dirname(__file__), 'ext', 'gitdb')) - - try: - import gitdb - except ImportError as e: - raise ImportError("'gitdb' could not be found in your PYTHONPATH") from e - # END verify import - -#} END initialization - - -################# -_init_externals() -################# - -#{ Imports - -try: - from git.config import GitConfigParser # @NoMove @IgnorePep8 - from git.objects import * # @NoMove @IgnorePep8 - from git.refs import * # @NoMove @IgnorePep8 - from git.diff import * # @NoMove @IgnorePep8 - from git.db import * # @NoMove @IgnorePep8 - from git.cmd import Git # @NoMove @IgnorePep8 - from git.repo import Repo # @NoMove @IgnorePep8 - from git.remote import * # @NoMove @IgnorePep8 - from git.index import * # @NoMove @IgnorePep8 - from git.util import ( # @NoMove @IgnorePep8 - LockFile, - BlockingLockFile, - Stats, - Actor, - rmtree, - ) -except GitError as exc: - raise ImportError('%s: %s' % (exc.__class__.__name__, exc)) from exc - -#} END imports - -__all__ = [name for name, obj in locals().items() - if not (name.startswith('_') or inspect.ismodule(obj))] - - -#{ Initialize git executable path -GIT_OK = None - - -def refresh(path: Optional[PathLike] = None) -> None: - """Convenience method for setting the git executable path.""" - global GIT_OK - GIT_OK = False - - if not Git.refresh(path=path): - return - if not FetchInfo.refresh(): - return - - GIT_OK = True -#} END initialize git executable path - - -################# -try: - refresh() -except Exception as exc: - raise ImportError('Failed to initialize: {0}'.format(exc)) from exc -################# diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/__init__.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index e2e2b4c..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/cmd.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/cmd.cpython-38.pyc deleted file mode 100644 index 277cbb2..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/cmd.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/compat.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/compat.cpython-38.pyc deleted file mode 100644 index d151c6e..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/compat.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/config.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/config.cpython-38.pyc deleted file mode 100644 index 7d06fe7..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/config.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/db.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/db.cpython-38.pyc deleted file mode 100644 index ad8e6ff..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/db.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/diff.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/diff.cpython-38.pyc deleted file mode 100644 index 6a718b5..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/diff.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/exc.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/exc.cpython-38.pyc deleted file mode 100644 index c6b4520..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/exc.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/remote.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/remote.cpython-38.pyc deleted file mode 100644 index 196947c..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/remote.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/types.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/types.cpython-38.pyc deleted file mode 100644 index 191af27..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/types.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/__pycache__/util.cpython-38.pyc b/.venv/lib/python3.8/site-packages/git/__pycache__/util.cpython-38.pyc deleted file mode 100644 index 440b14b..0000000 Binary files a/.venv/lib/python3.8/site-packages/git/__pycache__/util.cpython-38.pyc and /dev/null differ diff --git a/.venv/lib/python3.8/site-packages/git/cmd.py b/.venv/lib/python3.8/site-packages/git/cmd.py deleted file mode 100644 index 8fb1074..0000000 --- a/.venv/lib/python3.8/site-packages/git/cmd.py +++ /dev/null @@ -1,1285 +0,0 @@ -# cmd.py -# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors -# -# This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php -from __future__ import annotations -from contextlib import contextmanager -import io -import logging -import os -import signal -from subprocess import ( - call, - Popen, - PIPE -) -import subprocess -import threading -from textwrap import dedent - -from git.compat import ( - defenc, - force_bytes, - safe_decode, - is_posix, - is_win, -) -from git.exc import CommandError -from git.util import is_cygwin_git, cygpath, expand_path, remove_password_if_present - -from .exc import ( - GitCommandError, - GitCommandNotFound -) -from .util import ( - LazyMixin, - stream_copy, -) - -# typing --------------------------------------------------------------------------- - -from typing import (Any, AnyStr, BinaryIO, Callable, Dict, IO, Iterator, List, Mapping, - Sequence, TYPE_CHECKING, TextIO, Tuple, Union, cast, overload) - -from git.types import PathLike, Literal, TBD - -if TYPE_CHECKING: - from git.repo.base import Repo - from git.diff import DiffIndex - - -# --------------------------------------------------------------------------------- - -execute_kwargs = {'istream', 'with_extended_output', - 'with_exceptions', 'as_process', 'stdout_as_string', - 'output_stream', 'with_stdout', 'kill_after_timeout', - 'universal_newlines', 'shell', 'env', 'max_chunk_size'} - -log = logging.getLogger(__name__) -log.addHandler(logging.NullHandler()) - -__all__ = ('Git',) - - -# ============================================================================== -## @name Utilities -# ------------------------------------------------------------------------------ -# Documentation -## @{ - -def handle_process_output(process: 'Git.AutoInterrupt' | Popen, - stdout_handler: Union[None, - Callable[[AnyStr], None], - Callable[[List[AnyStr]], None], - Callable[[bytes, 'Repo', 'DiffIndex'], None]], - stderr_handler: Union[None, - Callable[[AnyStr], None], - Callable[[List[AnyStr]], None]], - finalizer: Union[None, - Callable[[Union[subprocess.Popen, 'Git.AutoInterrupt']], None]] = None, - decode_streams: bool = True, - kill_after_timeout: Union[None, float] = None) -> None: - """Registers for notifications to learn that process output is ready to read, and dispatches lines to - the respective line handlers. - This function returns once the finalizer returns - - :return: result of finalizer - :param process: subprocess.Popen instance - :param stdout_handler: f(stdout_line_string), or None - :param stderr_handler: f(stderr_line_string), or None - :param finalizer: f(proc) - wait for proc to finish - :param decode_streams: - Assume stdout/stderr streams are binary and decode them before pushing \ - their contents to handlers. - Set it to False if `universal_newline == True` (then streams are in text-mode) - or if decoding must happen later (i.e. for Diffs). - :param kill_after_timeout: - float or None, Default = None - To specify a timeout in seconds for the git command, after which the process - should be killed. - """ - # Use 2 "pump" threads and wait for both to finish. - def pump_stream(cmdline: List[str], name: str, stream: Union[BinaryIO, TextIO], is_decode: bool, - handler: Union[None, Callable[[Union[bytes, str]], None]]) -> None: - try: - for line in stream: - if handler: - if is_decode: - assert isinstance(line, bytes) - line_str = line.decode(defenc) - handler(line_str) - else: - handler(line) - - except Exception as ex: - log.error(f"Pumping {name!r} of cmd({remove_password_if_present(cmdline)}) failed due to: {ex!r}") - if "I/O operation on closed file" not in str(ex): - # Only reraise if the error was not due to the stream closing - raise CommandError([f'<{name}-pump>'] + remove_password_if_present(cmdline), ex) from ex - finally: - stream.close() - - if hasattr(process, 'proc'): - process = cast('Git.AutoInterrupt', process) - cmdline: str | Tuple[str, ...] | List[str] = getattr(process.proc, 'args', '') - p_stdout = process.proc.stdout if process.proc else None - p_stderr = process.proc.stderr if process.proc else None - else: - process = cast(Popen, process) - cmdline = getattr(process, 'args', '') - p_stdout = process.stdout - p_stderr = process.stderr - - if not isinstance(cmdline, (tuple, list)): - cmdline = cmdline.split() - - pumps: List[Tuple[str, IO, Callable[..., None] | None]] = [] - if p_stdout: - pumps.append(('stdout', p_stdout, stdout_handler)) - if p_stderr: - pumps.append(('stderr', p_stderr, stderr_handler)) - - threads: List[threading.Thread] = [] - - for name, stream, handler in pumps: - t = threading.Thread(target=pump_stream, - args=(cmdline, name, stream, decode_streams, handler)) - t.daemon = True - t.start() - threads.append(t) - - ## FIXME: Why Join?? Will block if `stdin` needs feeding... - # - for t in threads: - t.join(timeout=kill_after_timeout) - if t.is_alive(): - if isinstance(process, Git.AutoInterrupt): - process._terminate() - else: # Don't want to deal with the other case - raise RuntimeError("Thread join() timed out in cmd.handle_process_output()." - f" kill_after_timeout={kill_after_timeout} seconds") - if stderr_handler: - error_str: Union[str, bytes] = ( - "error: process killed because it timed out." - f" kill_after_timeout={kill_after_timeout} seconds") - if not decode_streams and isinstance(p_stderr, BinaryIO): - # Assume stderr_handler needs binary input - error_str = cast(str, error_str) - error_str = error_str.encode() - # We ignore typing on the next line because mypy does not like - # the way we inferred that stderr takes str or bytes - stderr_handler(error_str) # type: ignore - - if finalizer: - return finalizer(process) - else: - return None - - -def dashify(string: str) -> str: - return string.replace('_', '-') - - -def slots_to_dict(self: object, exclude: Sequence[str] = ()) -> Dict[str, Any]: - return {s: getattr(self, s) for s in self.__slots__ if s not in exclude} - - -def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], excluded: Sequence[str] = ()) -> None: - for k, v in d.items(): - setattr(self, k, v) - for k in excluded: - setattr(self, k, None) - -## -- End Utilities -- @} - - -# value of Windows process creation flag taken from MSDN -CREATE_NO_WINDOW = 0x08000000 - -## CREATE_NEW_PROCESS_GROUP is needed to allow killing it afterwards, -# see https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal -PROC_CREATIONFLAGS = (CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore[attr-defined] - if is_win else 0) # mypy error if not windows - - -class Git(LazyMixin): - - """ - The Git class manages communication with the Git binary. - - It provides a convenient interface to calling the Git binary, such as in:: - - g = Git( git_dir ) - g.init() # calls 'git init' program - rval = g.ls_files() # calls 'git ls-files' program - - ``Debugging`` - Set the GIT_PYTHON_TRACE environment variable print each invocation - of the command to stdout. - Set its value to 'full' to see details about the returned values. - """ - __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version_info", - "_git_options", "_persistent_git_options", "_environment") - - _excluded_ = ('cat_file_all', 'cat_file_header', '_version_info') - - def __getstate__(self) -> Dict[str, Any]: - return slots_to_dict(self, exclude=self._excluded_) - - def __setstate__(self, d: Dict[str, Any]) -> None: - dict_to_slots_and__excluded_are_none(self, d, excluded=self._excluded_) - - # CONFIGURATION - - git_exec_name = "git" # default that should work on linux and windows - - # Enables debugging of GitPython's git commands - GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) - - # If True, a shell will be used when executing git commands. - # This should only be desirable on Windows, see https://github.com/gitpython-developers/GitPython/pull/126 - # and check `git/test_repo.py:TestRepo.test_untracked_files()` TC for an example where it is required. - # Override this value using `Git.USE_SHELL = True` - USE_SHELL = False - - # Provide the full path to the git executable. Otherwise it assumes git is in the path - _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" - _refresh_env_var = "GIT_PYTHON_REFRESH" - GIT_PYTHON_GIT_EXECUTABLE = None - # note that the git executable is actually found during the refresh step in - # the top level __init__ - - @classmethod - def refresh(cls, path: Union[None, PathLike] = None) -> bool: - """This gets called by the refresh function (see the top level - __init__). - """ - # discern which path to refresh with - if path is not None: - new_git = os.path.expanduser(path) - new_git = os.path.abspath(new_git) - else: - new_git = os.environ.get(cls._git_exec_env_var, cls.git_exec_name) - - # keep track of the old and new git executable path - old_git = cls.GIT_PYTHON_GIT_EXECUTABLE - cls.GIT_PYTHON_GIT_EXECUTABLE = new_git - - # test if the new git executable path is valid - - # - a GitCommandNotFound error is spawned by ourselves - # - a PermissionError is spawned if the git executable provided - # cannot be executed for whatever reason - - has_git = False - try: - cls().version() - has_git = True - except (GitCommandNotFound, PermissionError): - pass - - # warn or raise exception if test failed - if not has_git: - err = dedent("""\ - Bad git executable. - The git executable must be specified in one of the following ways: - - be included in your $PATH - - be set via $%s - - explicitly set via git.refresh() - """) % cls._git_exec_env_var - - # revert to whatever the old_git was - cls.GIT_PYTHON_GIT_EXECUTABLE = old_git - - if old_git is None: - # on the first refresh (when GIT_PYTHON_GIT_EXECUTABLE is - # None) we only are quiet, warn, or error depending on the - # GIT_PYTHON_REFRESH value - - # determine what the user wants to happen during the initial - # refresh we expect GIT_PYTHON_REFRESH to either be unset or - # be one of the following values: - # 0|q|quiet|s|silence - # 1|w|warn|warning - # 2|r|raise|e|error - - mode = os.environ.get(cls._refresh_env_var, "raise").lower() - - quiet = ["quiet", "q", "silence", "s", "none", "n", "0"] - warn = ["warn", "w", "warning", "1"] - error = ["error", "e", "raise", "r", "2"] - - if mode in quiet: - pass - elif mode in warn or mode in error: - err = dedent("""\ - %s - All git commands will error until this is rectified. - - This initial warning can be silenced or aggravated in the future by setting the - $%s environment variable. Use one of the following values: - - %s: for no warning or exception - - %s: for a printed warning - - %s: for a raised exception - - Example: - export %s=%s - """) % ( - err, - cls._refresh_env_var, - "|".join(quiet), - "|".join(warn), - "|".join(error), - cls._refresh_env_var, - quiet[0]) - - if mode in warn: - print("WARNING: %s" % err) - else: - raise ImportError(err) - else: - err = dedent("""\ - %s environment variable has been set but it has been set with an invalid value. - - Use only the following values: - - %s: for no warning or exception - - %s: for a printed warning - - %s: for a raised exception - """) % ( - cls._refresh_env_var, - "|".join(quiet), - "|".join(warn), - "|".join(error)) - raise ImportError(err) - - # we get here if this was the init refresh and the refresh mode - # was not error, go ahead and set the GIT_PYTHON_GIT_EXECUTABLE - # such that we discern the difference between a first import - # and a second import - cls.GIT_PYTHON_GIT_EXECUTABLE = cls.git_exec_name - else: - # after the first refresh (when GIT_PYTHON_GIT_EXECUTABLE - # is no longer None) we raise an exception - raise GitCommandNotFound("git", err) - - return has_git - - @classmethod - def is_cygwin(cls) -> bool: - return is_cygwin_git(cls.GIT_PYTHON_GIT_EXECUTABLE) - - @overload - @classmethod - def polish_url(cls, url: str, is_cygwin: Literal[False] = ...) -> str: - ... - - @overload - @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: - ... - - @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: - if is_cygwin is None: - is_cygwin = cls.is_cygwin() - - if is_cygwin: - url = cygpath(url) - else: - """Remove any backslahes from urls to be written in config files. - - Windows might create config-files containing paths with backslashed, - but git stops liking them as it will escape the backslashes. - Hence we undo the escaping just to be sure. - """ - url = os.path.expandvars(url) - if url.startswith('~'): - url = os.path.expanduser(url) - url = url.replace("\\\\", "\\").replace("\\", "/") - return url - - class AutoInterrupt(object): - """Kill/Interrupt the stored process instance once this instance goes out of scope. It is - used to prevent processes piling up in case iterators stop reading. - Besides all attributes are wired through to the contained process object. - - The wait method was overridden to perform automatic status code checking - and possibly raise.""" - - __slots__ = ("proc", "args", "status") - - # If this is non-zero it will override any status code during - # _terminate, used to prevent race conditions in testing - _status_code_if_terminate: int = 0 - - def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: - self.proc = proc - self.args = args - self.status: Union[int, None] = None - - def _terminate(self) -> None: - """Terminate the underlying process""" - if self.proc is None: - return - - proc = self.proc - self.proc = None - if proc.stdin: - proc.stdin.close() - if proc.stdout: - proc.stdout.close() - if proc.stderr: - proc.stderr.close() - # did the process finish already so we have a return code ? - try: - if proc.poll() is not None: - self.status = self._status_code_if_terminate or proc.poll() - return None - except OSError as ex: - log.info("Ignored error after process had died: %r", ex) - - # can be that nothing really exists anymore ... - if os is None or getattr(os, 'kill', None) is None: - return None - - # try to kill it - try: - proc.terminate() - status = proc.wait() # ensure process goes away - - self.status = self._status_code_if_terminate or status - except OSError as ex: - log.info("Ignored error after process had died: %r", ex) - except AttributeError: - # try windows - # for some reason, providing None for stdout/stderr still prints something. This is why - # we simply use the shell and redirect to nul. Its slower than CreateProcess, question - # is whether we really want to see all these messages. Its annoying no matter what. - if is_win: - call(("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(proc.pid)), shell=True) - # END exception handling - - def __del__(self) -> None: - self._terminate() - - def __getattr__(self, attr: str) -> Any: - return getattr(self.proc, attr) - - # TODO: Bad choice to mimic `proc.wait()` but with different args. - def wait(self, stderr: Union[None, str, bytes] = b'') -> int: - """Wait for the process and return its status code. - - :param stderr: Previously read value of stderr, in case stderr is already closed. - :warn: may deadlock if output or error pipes are used and not handled separately. - :raise GitCommandError: if the return status is not 0""" - if stderr is None: - stderr_b = b'' - stderr_b = force_bytes(data=stderr, encoding='utf-8') - status: Union[int, None] - if self.proc is not None: - status = self.proc.wait() - p_stderr = self.proc.stderr - else: # Assume the underlying proc was killed earlier or never existed - status = self.status - p_stderr = None - - def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: - if stream: - try: - return stderr_b + force_bytes(stream.read()) - except ValueError: - return stderr_b or b'' - else: - return stderr_b or b'' - - # END status handling - - if status != 0: - errstr = read_all_from_possibly_closed_stream(p_stderr) - log.debug('AutoInterrupt wait stderr: %r' % (errstr,)) - raise GitCommandError(remove_password_if_present(self.args), status, errstr) - return status - - # END auto interrupt - - class CatFileContentStream(object): - - """Object representing a sized read-only stream returning the contents of - an object. - It behaves like a stream, but counts the data read and simulates an empty - stream once our sized content region is empty. - If not all data is read to the end of the objects's lifetime, we read the - rest to assure the underlying stream continues to work""" - - __slots__: Tuple[str, ...] = ('_stream', '_nbr', '_size') - - def __init__(self, size: int, stream: IO[bytes]) -> None: - self._stream = stream - self._size = size - self._nbr = 0 # num bytes read - - # special case: if the object is empty, has null bytes, get the - # final newline right away. - if size == 0: - stream.read(1) - # END handle empty streams - - def read(self, size: int = -1) -> bytes: - bytes_left = self._size - self._nbr - if bytes_left == 0: - return b'' - if size > -1: - # assure we don't try to read past our limit - size = min(bytes_left, size) - else: - # they try to read all, make sure its not more than what remains - size = bytes_left - # END check early depletion - data = self._stream.read(size) - self._nbr += len(data) - - # check for depletion, read our final byte to make the stream usable by others - if self._size - self._nbr == 0: - self._stream.read(1) # final newline - # END finish reading - return data - - def readline(self, size: int = -1) -> bytes: - if self._nbr == self._size: - return b'' - - # clamp size to lowest allowed value - bytes_left = self._size - self._nbr - if size > -1: - size = min(bytes_left, size) - else: - size = bytes_left - # END handle size - - data = self._stream.readline(size) - self._nbr += len(data) - - # handle final byte - if self._size - self._nbr == 0: - self._stream.read(1) - # END finish reading - - return data - - def readlines(self, size: int = -1) -> List[bytes]: - if self._nbr == self._size: - return [] - - # leave all additional logic to our readline method, we just check the size - out = [] - nbr = 0 - while True: - line = self.readline() - if not line: - break - out.append(line) - if size > -1: - nbr += len(line) - if nbr > size: - break - # END handle size constraint - # END readline loop - return out - - # skipcq: PYL-E0301 - def __iter__(self) -> 'Git.CatFileContentStream': - return self - - def __next__(self) -> bytes: - return next(self) - - def next(self) -> bytes: - line = self.readline() - if not line: - raise StopIteration - - return line - - def __del__(self) -> None: - bytes_left = self._size - self._nbr - if bytes_left: - # read and discard - seeking is impossible within a stream - # includes terminating newline - self._stream.read(bytes_left + 1) - # END handle incomplete read - - def __init__(self, working_dir: Union[None, PathLike] = None): - """Initialize this instance with: - - :param working_dir: - Git directory we should work in. If None, we always work in the current - directory as returned by os.getcwd(). - It is meant to be the working tree directory if available, or the - .git directory in case of bare repositories.""" - super(Git, self).__init__() - self._working_dir = expand_path(working_dir) - self._git_options: Union[List[str], Tuple[str, ...]] = () - self._persistent_git_options: List[str] = [] - - # Extra environment variables to pass to git commands - self._environment: Dict[str, str] = {} - - # cached command slots - self.cat_file_header: Union[None, TBD] = None - self.cat_file_all: Union[None, TBD] = None - - def __getattr__(self, name: str) -> Any: - """A convenience method as it allows to call the command as if it was - an object. - :return: Callable object that will execute call _call_process with your arguments.""" - if name[0] == '_': - return LazyMixin.__getattr__(self, name) - return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) - - def set_persistent_git_options(self, **kwargs: Any) -> None: - """Specify command line options to the git executable - for subsequent subcommand calls - - :param kwargs: - is a dict of keyword arguments. - these arguments are passed as in _call_process - but will be passed to the git command rather than - the subcommand. - """ - - self._persistent_git_options = self.transform_kwargs( - split_single_char_options=True, **kwargs) - - def _set_cache_(self, attr: str) -> None: - if attr == '_version_info': - # We only use the first 4 numbers, as everything else could be strings in fact (on windows) - process_version = self._call_process('version') # should be as default *args and **kwargs used - version_numbers = process_version.split(' ')[2] - - self._version_info = cast(Tuple[int, int, int, int], - tuple(int(n) for n in version_numbers.split('.')[:4] if n.isdigit()) - ) - else: - super(Git, self)._set_cache_(attr) - # END handle version info - - @ property - def working_dir(self) -> Union[None, PathLike]: - """:return: Git directory we are working on""" - return self._working_dir - - @ property - def version_info(self) -> Tuple[int, int, int, int]: - """ - :return: tuple(int, int, int, int) tuple with integers representing the major, minor - and additional version numbers as parsed from git version. - This value is generated on demand and is cached""" - return self._version_info - - @ overload - def execute(self, - command: Union[str, Sequence[Any]], - *, - as_process: Literal[True] - ) -> 'AutoInterrupt': - ... - - @ overload - def execute(self, - command: Union[str, Sequence[Any]], - *, - as_process: Literal[False] = False, - stdout_as_string: Literal[True] - ) -> Union[str, Tuple[int, str, str]]: - ... - - @ overload - def execute(self, - command: Union[str, Sequence[Any]], - *, - as_process: Literal[False] = False, - stdout_as_string: Literal[False] = False - ) -> Union[bytes, Tuple[int, bytes, str]]: - ... - - @ overload - def execute(self, - command: Union[str, Sequence[Any]], - *, - with_extended_output: Literal[False], - as_process: Literal[False], - stdout_as_string: Literal[True] - ) -> str: - ... - - @ overload - def execute(self, - command: Union[str, Sequence[Any]], - *, - with_extended_output: Literal[False], - as_process: Literal[False], - stdout_as_string: Literal[False] - ) -> bytes: - ... - - def execute(self, - command: Union[str, Sequence[Any]], - istream: Union[None, BinaryIO] = None, - with_extended_output: bool = False, - with_exceptions: bool = True, - as_process: bool = False, - output_stream: Union[None, BinaryIO] = None, - stdout_as_string: bool = True, - kill_after_timeout: Union[None, float] = None, - with_stdout: bool = True, - universal_newlines: bool = False, - shell: Union[None, bool] = None, - env: Union[None, Mapping[str, str]] = None, - max_chunk_size: int = io.DEFAULT_BUFFER_SIZE, - **subprocess_kwargs: Any - ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], AutoInterrupt]: - """Handles executing the command on the shell and consumes and returns - the returned information (stdout) - - :param command: - The command argument list to execute. - It should be a string, or a sequence of program arguments. The - program to execute is the first item in the args sequence or string. - - :param istream: - Standard input filehandle passed to subprocess.Popen. - - :param with_extended_output: - Whether to return a (status, stdout, stderr) tuple. - - :param with_exceptions: - Whether to raise an exception when git returns a non-zero status. - - :param as_process: - Whether to return the created process instance directly from which - streams can be read on demand. This will render with_extended_output and - with_exceptions ineffective - the caller will have - to deal with the details himself. - It is important to note that the process will be placed into an AutoInterrupt - wrapper that will interrupt the process once it goes out of scope. If you - use the command in iterators, you should pass the whole process instance - instead of a single stream. - - :param output_stream: - If set to a file-like object, data produced by the git command will be - output to the given stream directly. - This feature only has any effect if as_process is False. Processes will - always be created with a pipe due to issues with subprocess. - This merely is a workaround as data will be copied from the - output pipe to the given output stream directly. - Judging from the implementation, you shouldn't use this flag ! - - :param stdout_as_string: - if False, the commands standard output will be bytes. Otherwise, it will be - decoded into a string using the default encoding (usually utf-8). - The latter can fail, if the output contains binary data. - - :param env: - A dictionary of environment variables to be passed to `subprocess.Popen`. - - :param max_chunk_size: - Maximum number of bytes in one chunk of data passed to the output_stream in - one invocation of write() method. If the given number is not positive then - the default value is used. - - :param subprocess_kwargs: - Keyword arguments to be passed to subprocess.Popen. Please note that - some of the valid kwargs are already set by this method, the ones you - specify may not be the same ones. - - :param with_stdout: If True, default True, we open stdout on the created process - :param universal_newlines: - if True, pipes will be opened as text, and lines are split at - all known line endings. - :param shell: - Whether to invoke commands through a shell (see `Popen(..., shell=True)`). - It overrides :attr:`USE_SHELL` if it is not `None`. - :param kill_after_timeout: - To specify a timeout in seconds for the git command, after which the process - should be killed. This will have no effect if as_process is set to True. It is - set to None by default and will let the process run until the timeout is - explicitly specified. This feature is not supported on Windows. It's also worth - noting that kill_after_timeout uses SIGKILL, which can have negative side - effects on a repository. For example, stale locks in case of git gc could - render the repository incapable of accepting changes until the lock is manually - removed. - - :return: - * str(output) if extended_output = False (Default) - * tuple(int(status), str(stdout), str(stderr)) if extended_output = True - - if output_stream is True, the stdout value will be your output stream: - * output_stream if extended_output = False - * tuple(int(status), output_stream, str(stderr)) if extended_output = True - - Note git is executed with LC_MESSAGES="C" to ensure consistent - output regardless of system language. - - :raise GitCommandError: - - :note: - If you add additional keyword arguments to the signature of this method, - you must update the execute_kwargs tuple housed in this module.""" - # Remove password for the command if present - redacted_command = remove_password_if_present(command) - if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != 'full' or as_process): - log.info(' '.join(redacted_command)) - - # Allow the user to have the command executed in their working dir. - try: - cwd = self._working_dir or os.getcwd() # type: Union[None, str] - if not os.access(str(cwd), os.X_OK): - cwd = None - except FileNotFoundError: - cwd = None - - # Start the process - inline_env = env - env = os.environ.copy() - # Attempt to force all output to plain ascii english, which is what some parsing code - # may expect. - # According to stackoverflow (http://goo.gl/l74GC8), we are setting LANGUAGE as well - # just to be sure. - env["LANGUAGE"] = "C" - env["LC_ALL"] = "C" - env.update(self._environment) - if inline_env is not None: - env.update(inline_env) - - if is_win: - cmd_not_found_exception = OSError - if kill_after_timeout is not None: - raise GitCommandError(redacted_command, '"kill_after_timeout" feature is not supported on Windows.') - else: - cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable - # end handle - - stdout_sink = (PIPE - if with_stdout - else getattr(subprocess, 'DEVNULL', None) or open(os.devnull, 'wb')) - istream_ok = "None" - if istream: - istream_ok = "" - log.debug("Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)", - redacted_command, cwd, universal_newlines, shell, istream_ok) - try: - proc = Popen(command, - env=env, - cwd=cwd, - bufsize=-1, - stdin=istream, - stderr=PIPE, - stdout=stdout_sink, - shell=shell is not None and shell or self.USE_SHELL, - close_fds=is_posix, # unsupported on windows - universal_newlines=universal_newlines, - creationflags=PROC_CREATIONFLAGS, - **subprocess_kwargs - ) - - except cmd_not_found_exception as err: - raise GitCommandNotFound(redacted_command, err) from err - else: - # replace with a typeguard for Popen[bytes]? - proc.stdout = cast(BinaryIO, proc.stdout) - proc.stderr = cast(BinaryIO, proc.stderr) - - if as_process: - return self.AutoInterrupt(proc, command) - - def _kill_process(pid: int) -> None: - """ Callback method to kill a process. """ - p = Popen(['ps', '--ppid', str(pid)], stdout=PIPE, - creationflags=PROC_CREATIONFLAGS) - child_pids = [] - if p.stdout is not None: - for line in p.stdout: - if len(line.split()) > 0: - local_pid = (line.split())[0] - if local_pid.isdigit(): - child_pids.append(int(local_pid)) - try: - # Windows does not have SIGKILL, so use SIGTERM instead - sig = getattr(signal, 'SIGKILL', signal.SIGTERM) - os.kill(pid, sig) - for child_pid in child_pids: - try: - os.kill(child_pid, sig) - except OSError: - pass - kill_check.set() # tell the main routine that the process was killed - except OSError: - # It is possible that the process gets completed in the duration after timeout - # happens and before we try to kill the process. - pass - return - # end - - if kill_after_timeout is not None: - kill_check = threading.Event() - watchdog = threading.Timer(kill_after_timeout, _kill_process, args=(proc.pid,)) - - # Wait for the process to return - status = 0 - stdout_value: Union[str, bytes] = b'' - stderr_value: Union[str, bytes] = b'' - newline = "\n" if universal_newlines else b"\n" - try: - if output_stream is None: - if kill_after_timeout is not None: - watchdog.start() - stdout_value, stderr_value = proc.communicate() - if kill_after_timeout is not None: - watchdog.cancel() - if kill_check.is_set(): - stderr_value = ('Timeout: the command "%s" did not complete in %d ' - 'secs.' % (" ".join(redacted_command), kill_after_timeout)) - if not universal_newlines: - stderr_value = stderr_value.encode(defenc) - # strip trailing "\n" - if stdout_value.endswith(newline): # type: ignore - stdout_value = stdout_value[:-1] - if stderr_value.endswith(newline): # type: ignore - stderr_value = stderr_value[:-1] - - status = proc.returncode - else: - max_chunk_size = max_chunk_size if max_chunk_size and max_chunk_size > 0 else io.DEFAULT_BUFFER_SIZE - stream_copy(proc.stdout, output_stream, max_chunk_size) - stdout_value = proc.stdout.read() - stderr_value = proc.stderr.read() - # strip trailing "\n" - if stderr_value.endswith(newline): # type: ignore - stderr_value = stderr_value[:-1] - status = proc.wait() - # END stdout handling - finally: - proc.stdout.close() - proc.stderr.close() - - if self.GIT_PYTHON_TRACE == 'full': - cmdstr = " ".join(redacted_command) - - def as_text(stdout_value: Union[bytes, str]) -> str: - return not output_stream and safe_decode(stdout_value) or '' - # end - - if stderr_value: - log.info("%s -> %d; stdout: '%s'; stderr: '%s'", - cmdstr, status, as_text(stdout_value), safe_decode(stderr_value)) - elif stdout_value: - log.info("%s -> %d; stdout: '%s'", cmdstr, status, as_text(stdout_value)) - else: - log.info("%s -> %d", cmdstr, status) - # END handle debug printing - - if with_exceptions and status != 0: - raise GitCommandError(redacted_command, status, stderr_value, stdout_value) - - if isinstance(stdout_value, bytes) and stdout_as_string: # could also be output_stream - stdout_value = safe_decode(stdout_value) - - # Allow access to the command's status code - if with_extended_output: - return (status, stdout_value, safe_decode(stderr_value)) - else: - return stdout_value - - def environment(self) -> Dict[str, str]: - return self._environment - - def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: - """ - Set environment variables for future git invocations. Return all changed - values in a format that can be passed back into this function to revert - the changes: - - ``Examples``:: - - old_env = self.update_environment(PWD='/tmp') - self.update_environment(**old_env) - - :param kwargs: environment variables to use for git processes - :return: dict that maps environment variables to their old values - """ - old_env = {} - for key, value in kwargs.items(): - # set value if it is None - if value is not None: - old_env[key] = self._environment.get(key) - self._environment[key] = value - # remove key from environment if its value is None - elif key in self._environment: - old_env[key] = self._environment[key] - del self._environment[key] - return old_env - - @contextmanager - def custom_environment(self, **kwargs: Any) -> Iterator[None]: - """ - A context manager around the above ``update_environment`` method to restore the - environment back to its previous state after operation. - - ``Examples``:: - - with self.custom_environment(GIT_SSH='/bin/ssh_wrapper'): - repo.remotes.origin.fetch() - - :param kwargs: see update_environment - """ - old_env = self.update_environment(**kwargs) - try: - yield - finally: - self.update_environment(**old_env) - - def transform_kwarg(self, name: str, value: Any, split_single_char_options: bool) -> List[str]: - if len(name) == 1: - if value is True: - return ["-%s" % name] - elif value not in (False, None): - if split_single_char_options: - return ["-%s" % name, "%s" % value] - else: - return ["-%s%s" % (name, value)] - else: - if value is True: - return ["--%s" % dashify(name)] - elif value is not False and value is not None: - return ["--%s=%s" % (dashify(name), value)] - return [] - - def transform_kwargs(self, split_single_char_options: bool = True, **kwargs: Any) -> List[str]: - """Transforms Python style kwargs into git command line options.""" - args = [] - for k, v in kwargs.items(): - if isinstance(v, (list, tuple)): - for value in v: - args += self.transform_kwarg(k, value, split_single_char_options) - else: - args += self.transform_kwarg(k, v, split_single_char_options) - return args - - @classmethod - def __unpack_args(cls, arg_list: Sequence[str]) -> List[str]: - - outlist = [] - if isinstance(arg_list, (list, tuple)): - for arg in arg_list: - outlist.extend(cls.__unpack_args(arg)) - else: - outlist.append(str(arg_list)) - - return outlist - - def __call__(self, **kwargs: Any) -> 'Git': - """Specify command line options to the git executable - for a subcommand call - - :param kwargs: - is a dict of keyword arguments. - these arguments are passed as in _call_process - but will be passed to the git command rather than - the subcommand. - - ``Examples``:: - git(work_tree='/tmp').difftool()""" - self._git_options = self.transform_kwargs( - split_single_char_options=True, **kwargs) - return self - - @overload - def _call_process(self, method: str, *args: None, **kwargs: None - ) -> str: - ... # if no args given, execute called with all defaults - - @overload - def _call_process(self, method: str, - istream: int, - as_process: Literal[True], - *args: Any, **kwargs: Any - ) -> 'Git.AutoInterrupt': ... - - @overload - def _call_process(self, method: str, *args: Any, **kwargs: Any - ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], 'Git.AutoInterrupt']: - ... - - def _call_process(self, method: str, *args: Any, **kwargs: Any - ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], 'Git.AutoInterrupt']: - """Run the given git command with the specified arguments and return - the result as a String - - :param method: - is the command. Contained "_" characters will be converted to dashes, - such as in 'ls_files' to call 'ls-files'. - - :param args: - is the list of arguments. If None is included, it will be pruned. - This allows your commands to call git more conveniently as None - is realized as non-existent - - :param kwargs: - It contains key-values for the following: - - the :meth:`execute()` kwds, as listed in :var:`execute_kwargs`; - - "command options" to be converted by :meth:`transform_kwargs()`; - - the `'insert_kwargs_after'` key which its value must match one of ``*args`` - and any cmd-options will be appended after the matched arg. - - Examples:: - - git.rev_list('master', max_count=10, header=True) - - turns into:: - - git rev-list max-count 10 --header master - - :return: Same as ``execute`` - if no args given used execute default (esp. as_process = False, stdout_as_string = True) - and return str """ - # Handle optional arguments prior to calling transform_kwargs - # otherwise these'll end up in args, which is bad. - exec_kwargs = {k: v for k, v in kwargs.items() if k in execute_kwargs} - opts_kwargs = {k: v for k, v in kwargs.items() if k not in execute_kwargs} - - insert_after_this_arg = opts_kwargs.pop('insert_kwargs_after', None) - - # Prepare the argument list - - opt_args = self.transform_kwargs(**opts_kwargs) - ext_args = self.__unpack_args([a for a in args if a is not None]) - - if insert_after_this_arg is None: - args_list = opt_args + ext_args - else: - try: - index = ext_args.index(insert_after_this_arg) - except ValueError as err: - raise ValueError("Couldn't find argument '%s' in args %s to insert cmd options after" - % (insert_after_this_arg, str(ext_args))) from err - # end handle error - args_list = ext_args[:index + 1] + opt_args + ext_args[index + 1:] - # end handle opts_kwargs - - call = [self.GIT_PYTHON_GIT_EXECUTABLE] - - # add persistent git options - call.extend(self._persistent_git_options) - - # add the git options, then reset to empty - # to avoid side_effects - call.extend(self._git_options) - self._git_options = () - - call.append(dashify(method)) - call.extend(args_list) - - return self.execute(call, **exec_kwargs) - - def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: - """ - :param header_line: - type_string size_as_int - - :return: (hex_sha, type_string, size_as_int) - - :raise ValueError: if the header contains indication for an error due to - incorrect input sha""" - tokens = header_line.split() - if len(tokens) != 3: - if not tokens: - raise ValueError("SHA could not be resolved, git returned: %r" % (header_line.strip())) - else: - raise ValueError("SHA %s could not be resolved, git returned: %r" % (tokens[0], header_line.strip())) - # END handle actual return value - # END error handling - - if len(tokens[0]) != 40: - raise ValueError("Failed to parse header: %r" % header_line) - return (tokens[0], tokens[1], int(tokens[2])) - - def _prepare_ref(self, ref: AnyStr) -> bytes: - # required for command to separate refs on stdin, as bytes - if isinstance(ref, bytes): - # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text - refstr: str = ref.decode('ascii') - elif not isinstance(ref, str): - refstr = str(ref) # could be ref-object - else: - refstr = ref - - if not refstr.endswith("\n"): - refstr += "\n" - return refstr.encode(defenc) - - def _get_persistent_cmd(self, attr_name: str, cmd_name: str, *args: Any, **kwargs: Any - ) -> 'Git.AutoInterrupt': - cur_val = getattr(self, attr_name) - if cur_val is not None: - return cur_val - - options = {"istream": PIPE, "as_process": True} - options.update(kwargs) - - cmd = self._call_process(cmd_name, *args, **options) - setattr(self, attr_name, cmd) - cmd = cast('Git.AutoInterrupt', cmd) - return cmd - - def __get_object_header(self, cmd: 'Git.AutoInterrupt', ref: AnyStr) -> Tuple[str, str, int]: - if cmd.stdin and cmd.stdout: - cmd.stdin.write(self._prepare_ref(ref)) - cmd.stdin.flush() - return self._parse_object_header(cmd.stdout.readline()) - else: - raise ValueError("cmd stdin was empty") - - def get_object_header(self, ref: str) -> Tuple[str, str, int]: - """ Use this method to quickly examine the type and size of the object behind - the given ref. - - :note: The method will only suffer from the costs of command invocation - once and reuses the command in subsequent calls. - - :return: (hexsha, type_string, size_as_int)""" - cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) - return self.__get_object_header(cmd, ref) - - def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: - """ As get_object_header, but returns object data as well - :return: (hexsha, type_string, size_as_int,data_string) - :note: not threadsafe""" - hexsha, typename, size, stream = self.stream_object_data(ref) - data = stream.read(size) - del(stream) - return (hexsha, typename, size, data) - - def stream_object_data(self, ref: str) -> Tuple[str, str, int, 'Git.CatFileContentStream']: - """ As get_object_header, but returns the data as a stream - - :return: (hexsha, type_string, size_as_int, stream) - :note: This method is not threadsafe, you need one independent Command instance per thread to be safe !""" - cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) - hexsha, typename, size = self.__get_object_header(cmd, ref) - cmd_stdout = cmd.stdout if cmd.stdout is not None else io.BytesIO() - return (hexsha, typename, size, self.CatFileContentStream(size, cmd_stdout)) - - def clear_cache(self) -> 'Git': - """Clear all kinds of internal caches to release resources. - - Currently persistent commands will be interrupted. - - :return: self""" - for cmd in (self.cat_file_all, self.cat_file_header): - if cmd: - cmd.__del__() - - self.cat_file_all = None - self.cat_file_header = None - return self diff --git a/.venv/lib/python3.8/site-packages/git/compat.py b/.venv/lib/python3.8/site-packages/git/compat.py deleted file mode 100644 index 988c04e..0000000 --- a/.venv/lib/python3.8/site-packages/git/compat.py +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- -# config.py -# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors -# -# This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php -"""utilities to help provide compatibility with python 3""" -# flake8: noqa - -import locale -import os -import sys - -from gitdb.utils.encoding import ( - force_bytes, # @UnusedImport - force_text # @UnusedImport -) - -# typing -------------------------------------------------------------------- - -from typing import ( - Any, - AnyStr, - Dict, - IO, - Optional, - Tuple, - Type, - Union, - overload, -) -# --------------------------------------------------------------------------- - - -is_win: bool = (os.name == 'nt') -is_posix = (os.name == 'posix') -is_darwin = (os.name == 'darwin') -defenc = sys.getfilesystemencoding() - - -@overload -def safe_decode(s: None) -> None: ... - - -@overload -def safe_decode(s: AnyStr) -> str: ... - - -def safe_decode(s: Union[AnyStr, None]) -> Optional[str]: - """Safely decodes a binary string to unicode""" - if isinstance(s, str): - return s - elif isinstance(s, bytes): - return s.decode(defenc, 'surrogateescape') - elif s is None: - return None - else: - raise TypeError('Expected bytes or text, but got %r' % (s,)) - - -@overload -def safe_encode(s: None) -> None: ... - - -@overload -def safe_encode(s: AnyStr) -> bytes: ... - - -def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: - """Safely encodes a binary string to unicode""" - if isinstance(s, str): - return s.encode(defenc) - elif isinstance(s, bytes): - return s - elif s is None: - return None - else: - raise TypeError('Expected bytes or text, but got %r' % (s,)) - - -@overload -def win_encode(s: None) -> None: ... - - -@overload -def win_encode(s: AnyStr) -> bytes: ... - - -def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: - """Encode unicodes for process arguments on Windows.""" - if isinstance(s, str): - return s.encode(locale.getpreferredencoding(False)) - elif isinstance(s, bytes): - return s - elif s is not None: - raise TypeError('Expected bytes or text, but got %r' % (s,)) - return None diff --git a/.venv/lib/python3.8/site-packages/git/config.py b/.venv/lib/python3.8/site-packages/git/config.py deleted file mode 100644 index cbd6602..0000000 --- a/.venv/lib/python3.8/site-packages/git/config.py +++ /dev/null @@ -1,847 +0,0 @@ -# config.py -# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors -# -# This module is part of GitPython and is released under -# the BSD License: http://www.opensource.org/licenses/bsd-license.php -"""Module containing module parser implementation able to properly read and write -configuration files""" - -import sys -import abc -from functools import wraps -import inspect -from io import BufferedReader, IOBase -import logging -import os -import re -import fnmatch - -from git.compat import ( - defenc, - force_text, - is_win, -) - -from git.util import LockFile - -import os.path as osp - -import configparser as cp - -# typing------------------------------------------------------- - -from typing import (Any, Callable, Generic, IO, List, Dict, Sequence, - TYPE_CHECKING, Tuple, TypeVar, Union, cast) - -from git.types import Lit_config_levels, ConfigLevels_Tup, PathLike, assert_never, _T - -if TYPE_CHECKING: - from git.repo.base import Repo - from io import BytesIO - -T_ConfigParser = TypeVar('T_ConfigParser', bound='GitConfigParser') -T_OMD_value = TypeVar('T_OMD_value', str, bytes, int, float, bool) - -if sys.version_info[:3] < (3, 7, 2): - # typing.Ordereddict not added until py 3.7.2 - from collections import OrderedDict - OrderedDict_OMD = OrderedDict -else: - from typing import OrderedDict - OrderedDict_OMD = OrderedDict[str, List[T_OMD_value]] # type: ignore[assignment, misc] - -# ------------------------------------------------------------- - -__all__ = ('GitConfigParser', 'SectionConstraint') - - -log = logging.getLogger('git.config') -log.addHandler(logging.NullHandler()) - -# invariants -# represents the configuration level of a configuration file - - -CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") - - -# Section pattern to detect conditional includes. -# https://git-scm.com/docs/git-config#_conditional_includes -CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") - - -class MetaParserBuilder(abc.ABCMeta): - """Utlity class wrapping base-class methods into decorators that assure read-only properties""" - def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> 'MetaParserBuilder': - """ - Equip all base-class methods with a needs_values decorator, and all non-const methods - with a set_dirty_and_flush_changes decorator in addition to that.""" - kmm = '_mutating_methods_' - if kmm in clsdict: - mutating_methods = clsdict[kmm] - for base in bases: - methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_")) - for name, method in methods: - if name in clsdict: - continue - method_with_values = needs_values(method) - if name in mutating_methods: - method_with_values = set_dirty_and_flush_changes(method_with_values) - # END mutating methods handling - - clsdict[name] = method_with_values - # END for each name/method pair - # END for each base - # END if mutating methods configuration is set - - new_type = super(MetaParserBuilder, cls).__new__(cls, name, bases, clsdict) - return new_type - - -def needs_values(func: Callable[..., _T]) -> Callable[..., _T]: - """Returns method assuring we read values (on demand) before we try to access them""" - - @wraps(func) - def assure_data_present(self: 'GitConfigParser', *args: Any, **kwargs: Any) -> _T: - self.read() - return func(self, *args, **kwargs) - # END wrapper method - return assure_data_present - - -def set_dirty_and_flush_changes(non_const_func: Callable[..., _T]) -> Callable[..., _T]: - """Return method that checks whether given non constant function may be called. - If so, the instance will be set dirty. - Additionally, we flush the changes right to disk""" - - def flush_changes(self: 'GitConfigParser', *args: Any, **kwargs: Any) -> _T: - rval = non_const_func(self, *args, **kwargs) - self._dirty = True - self.write() - return rval - # END wrapper method - flush_changes.__name__ = non_const_func.__name__ - return flush_changes - - -class SectionConstraint(Generic[T_ConfigParser]): - - """Constrains a ConfigParser to only option commands which are constrained to - always use the section we have been initialized with. - - It supports all ConfigParser methods that operate on an option. - - :note: - If used as a context manager, will release the wrapped ConfigParser.""" - __slots__ = ("_config", "_section_name") - _valid_attrs_ = ("get_value", "set_value", "get", "set", "getint", "getfloat", "getboolean", "has_option", - "remove_section", "remove_option", "options") - - def __init__(self, config: T_ConfigParser, section: str) -> None: - self._config = config - self._section_name = section - - def __del__(self) -> None: - # Yes, for some reason, we have to call it explicitly for it to work in PY3 ! - # Apparently __del__ doesn't get call anymore if refcount becomes 0 - # Ridiculous ... . - self._config.release() - - def __getattr__(self, attr: str) -> Any: - if attr in self._valid_attrs_: - return lambda *args, **kwargs: self._call_config(attr, *args, **kwargs) - return super(SectionConstraint, self).__getattribute__(attr) - - def _call_config(self, method: str, *args: Any, **kwargs: Any) -> Any: - """Call the configuration at the given method which must take a section name - as first argument""" - return getattr(self._config, method)(self._section_name, *args, **kwargs) - - @property - def config(self) -> T_ConfigParser: - """return: Configparser instance we constrain""" - return self._config - - def release(self) -> None: - """Equivalent to GitConfigParser.release(), which is called on our underlying parser instance""" - return self._config.release() - - def __enter__(self) -> 'SectionConstraint[T_ConfigParser]': - self._config.__enter__() - return self - - def __exit__(self, exception_type: str, exception_value: str, traceback: str) -> None: - self._config.__exit__(exception_type, exception_value, traceback) - - -class _OMD(OrderedDict_OMD): - """Ordered multi-dict.""" - - def __setitem__(self, key: str, value: _T) -> None: - super(_OMD, self).__setitem__(key, [value]) - - def add(self, key: str, value: Any) -> None: - if key not in self: - super(_OMD, self).__setitem__(key, [value]) - return None - super(_OMD, self).__getitem__(key).append(value) - - def setall(self, key: str, values: List[_T]) -> None: - super(_OMD, self).__setitem__(key, values) - - def __getitem__(self, key: str) -> Any: - return super(_OMD, self).__getitem__(key)[-1] - - def getlast(self, key: str) -> Any: - return super(_OMD, self).__getitem__(key)[-1] - - def setlast(self, key: str, value: Any) -> None: - if key not in self: - super(_OMD, self).__setitem__(key, [value]) - return - - prior = super(_OMD, self).__getitem__(key) - prior[-1] = value - - def get(self, key: str, default: Union[_T, None] = None) -> Union[_T, None]: - return super(_OMD, self).get(key, [default])[-1] - - def getall(self, key: str) -> List[_T]: - return super(_OMD, self).__getitem__(key) - - def items(self) -> List[Tuple[str, _T]]: # type: ignore[override] - """List of (key, last value for key).""" - return [(k, self[k]) for k in self] - - def items_all(self) -> List[Tuple[str, List[_T]]]: - """List of (key, list of values for key).""" - return [(k, self.getall(k)) for k in self] - - -def get_config_path(config_level: Lit_config_levels) -> str: - - # we do not support an absolute path of the gitconfig on windows , - # use the global config instead - if is_win and config_level == "system": - config_level = "global" - - if config_level == "system": - return "/etc/gitconfig" - elif config_level == "user": - config_home = os.environ.get("XDG_CONFIG_HOME") or osp.join(os.environ.get("HOME", '~'), ".config") - return osp.normpath(osp.expanduser(osp.join(config_home, "git", "config"))) - elif config_level == "global": - return osp.normpath(osp.expanduser("~/.gitconfig")) - elif config_level == "repository": - raise ValueError("No repo to get repository configuration from. Use Repo._get_config_path") - else: - # Should not reach here. Will raise ValueError if does. Static typing will warn missing elifs - assert_never(config_level, # type: ignore[unreachable] - ValueError(f"Invalid configuration level: {config_level!r}")) - - -class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): - - """Implements specifics required to read git style configuration files. - - This variation behaves much like the git.config command such that the configuration - will be read on demand based on the filepath given during initialization. - - The changes will automatically be written once the instance goes out of scope, but - can be triggered manually as well. - - The configuration file will be locked if you intend to change values preventing other - instances to write concurrently. - - :note: - The config is case-sensitive even when queried, hence section and option names - must match perfectly. - If used as a context manager, will release the locked file.""" - - #{ Configuration - # The lock type determines the type of lock to use in new configuration readers. - # They must be compatible to the LockFile interface. - # A suitable alternative would be the BlockingLockFile - t_lock = LockFile - re_comment = re.compile(r'^\s*[#;]') - - #} END configuration - - optvalueonly_source = r'\s*(?P