From 80826776551ab47145ed93ba8512a91344c82d0b Mon Sep 17 00:00:00 2001 From: Tomas Gavenciak Date: Fri, 15 May 2015 21:26:23 +0200 Subject: [PATCH] Poznamky a zdroje k Dakos exportu a importu --- dakosdump/README.md | 136 ++++++++++++++++++++++ dakosdump/__init__.py | 0 {seminar/dksdump => dakosdump}/helpers.py | 0 dakosdump/oracle_dump | 54 +++++++++ 4 files changed, 190 insertions(+) create mode 100644 dakosdump/README.md create mode 100644 dakosdump/__init__.py rename {seminar/dksdump => dakosdump}/helpers.py (100%) create mode 100644 dakosdump/oracle_dump diff --git a/dakosdump/README.md b/dakosdump/README.md new file mode 100644 index 00000000..3507a757 --- /dev/null +++ b/dakosdump/README.md @@ -0,0 +1,136 @@ +# Poznámky k exportu a reimportu MaM dat # + +### Dakos Oracle dump - postup ### + +Na lib. stroji na KAMu spustit: + + ORACLE_HOME=/nfs/nikam/sw/oracle-10.2/ora_home_10.2/ LC_ALL=C AESOP_PASSWD= perl oracle_dump + +Dumpne vsechny tabulky krom SYS.* + + from seminar.dksdump.helpers import read_all_tables, matchrows, onerow + ts = read_all_tables('seminar/dksdump/oracle_dks_dump/') + +Načte a vypíše všechny tabulky. Yay! +Pozor! V základu mají všechny podivný 2. řádek s bordelem! (čísla 3, 12, ...) + +Import do SQLite3 pomocí `.mode tabs` a `.import DKSROOT.V_SKOLY.csv DKSROOT_V_SKOLY`. + +### Postup pomocí sqlite3 s Dakos daty ### + +`MM_RESENI`, `MM_RESENI#BODY`: + + + + + +### Konkrétní programy pro data (starší metoda) ### + +Používají ovvpfile pro načtení souborů. + +#### DAKOS ID pro nektere skoly #### + + for i in sk[1:]: + f = Skola.objects.filter(nazev=i['NAZEV'], izo=i['IZO']); + if len(f)==1: + f[0].dakos_id = i['ID']; f[0].save() + + +#### Resitel #### + + for i in ts['BW-resitelia-export']: + Resitel.objects.create(jmeno=i['jmeno'], prijmeni=i['prijmeni'], pohlavi_muz=(i['pohlavi']==u'mu\u017e'), + rok_maturity=int(i['rok_maturity']), email=i['email'], telefon=i['telefon'], zasilat=Resitel.ZASILAT_DOMU, + ulice=i['ulica']+' '+i['cislo'], mesto=i['obec'], psc=i['psc'], stat=i['stat'], dakos_id=i['rid'], + poznamka="{'skola': '%s', 'dakos_skola_id': '%s'}\n" % (i['skola'], i['dakos_skola_id'], )) + +Doplneni nekterych skol: + + ae = ts['Aesop-MaM-export-2014-10-20-mod'] + skoly = ts['Aesop-skoly-export-2015-05-14-mod'] + for i in ae: + r = Resitel.objects.get(jmeno=i['name'], prijmeni=i['surname']); + izo = i['school'].lstrip('izo:'); + aesop_id = [s for s in skoly if izo in repr(s)][0]['id-aesop']; + sch = Skola.objects.get(aesop_id='aesop:'+aesop_id); + r.skola=sch; r.save() + +#### Cisla #### + +Hlavne ze souboru `archiv/seznam.csv` +Navic nejaka data z DB, ale nic moc + + for i in ts['BW-archiv-cisla-copy-mod']: Cislo.objects.create(rocnik=Rocnik.objects.get(rocnik=int(i['rocnik'])), cislo=i['cislo']) + +#### Uzivatele #### + + for i in ts['MAMOPER.MM_PASSWD']: + if not User.objects.filter(username=i['LOGIN']): User.objects.create_user(i['LOGIN']) + +#### Problemy #### + +`MM_ZADANIA`: + + typtable={'1': Problem.TYP_ULOHA, '2':Problem.TYP_TEMA, '3':Problem.TYP_SERIAL} + for i in ts['MAMOPER.MM_ZADANIA']: + print i + c = Cislo.objects.get(cislo=i['CISLO'] if i['CISLO']!='9' else '1', rocnik__rocnik=int(i['ROCNIK'])) + opravovatel = User.objects.get(username=i['OPRAVUJE']) if i['OPRAVUJE'] else None + Problem.objects.create(nazev=i['NAZOV'], typ=typtable[i['TYP']], stav=Problem.STAV_ZADANY, opravovatel=opravovatel, + kod=i['ULOHA'], cislo_zadani=c, body=int(i['MAX_BODY']) if i['MAX_BODY'] else None, + dakos_id='ZAD:%s.%s.%s.%s'%(i['ROCNIK'], i['CISLO'], i['ULOHA'], i['TYP'],)) + +`MM_DOZ`: +Dekodovani stavu navrhu (MFIO - oblast, Tema, Konfera, Pouzita, Zamitnuta, Doplnit (?)) + + def dectag(x): return ["MFIOTKPZD"[bi] for bi in range(0, 9) if (int(x) & (1 << bi))] + from django.utils.html import escape + def transdate(s): + if '.' not in s: return None + d,m,r = s.split('.'); return datetime.date(int(r)+2000, int(m), int(d)) + for i in ts['MAMOPER.MM_DOZ']: + print i; + tags=dectag(i['ZARADENIE']); + autor = User.objects.get(username=i['ORG']) if i['ORG'] else None; + Problem.objects.create(nazev=i['NAZOV'], typ=Problem.TYP_TEMA if 'T' in tags else Problem.TYP_ULOHA, + stav=Problem.STAV_ZADANY if 'P' in tags else (Problem.STAV_SMAZANY if 'Z' in tags else Problem.STAV_NAVRH), + dakos_id='DOZ:%s'%(i['ID'], ), text_problemu_org=escape(i['ZADANIE']), timestamp=transdate(i['CAS'])) + +`MM_AZAD`: + + def romdec(rom): + if not rom: return 0 + if rom[0] == 'X': return 10 + romdec(rom[1:]) + if rom[-1] == 'X': return 10 - romdec(rom[:-1]) + if rom[0] == 'V': return 5 + romdec(rom[1:]) + if rom[-1] == 'V': return 5 - romdec(rom[:-1]) + if rom[0] == 'I': return 1 + romdec(rom[1:]) + assert False + for i in ts['MAMOPER.MM_AZAD'][1:]: + print i; + tags=dectag(i['ZAMERANIE']); + r = romdec(i['ROC_Z']); + c1 = Cislo.objects.filter(rocnik__rocnik=r, cislo=i['CIS_Z']); + c1 = c1[0] if len(c1) else None; + c2 = Cislo.objects.filter(rocnik__rocnik=r, cislo=i['CIS_V']); + c2 = c2[0] if len(c2) else None; + Problem.objects.create(nazev=i['NAZOV'], typ=Problem.TYP_TEMA if 'T' in tags else Problem.TYP_ULOHA, stav=Problem.STAV_ZADANY, + kod=i['ZADANIE'][1:], cislo_zadani=c1, cislo_reseni=c2, dakos_id='AZAD:%s'%(i['ID'], ), + text_problemu_org=escape(u'Abstrakt:\n'+i['ABSTRAKT'])) + for i in ts['MAMOPER.MM_AZAD'][1:]: + print i; tags=dectag(i['ZAMERANIE']); p = Problem.objects.get(dakos_id='AZAD:%s'%(i['ID'], )); + [p.zamereni.add(t) for t in tags if t in 'MFIO']; p.save() + +#### Reseni #### + +`MM_RIESENIA`: + + for i in ts['MAMOPER.MM_RIESENIA']: + tstp = datetime.datetime.fromtimestamp(int(i['DORUCENA'])); + print i, tstp; + r = Resitel.objects.get(dakos_id=i['RIESITEL']); + u = Problem.objects.get(dakos_id="ZAD:%s.%s.%s.%s" % (i['ROCNIK'], i['CISLO'], i['ULOHA'], i['TYP'],)); + Reseni.objects.create(problem=u, resitel=r, body=int(i['POCET_BODOV'] or '0'), + poznamka='Zdroj: MM_RIESENIA\nForma: %s\n%s' % ({'1':'E', '2':'P'}[i['FORMA']], i['POZNAMKA'], ), timestamp=tstp) + + diff --git a/dakosdump/__init__.py b/dakosdump/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/seminar/dksdump/helpers.py b/dakosdump/helpers.py similarity index 100% rename from seminar/dksdump/helpers.py rename to dakosdump/helpers.py diff --git a/dakosdump/oracle_dump b/dakosdump/oracle_dump new file mode 100644 index 00000000..9653539f --- /dev/null +++ b/dakosdump/oracle_dump @@ -0,0 +1,54 @@ +#!/usr/bin/perl + +# RUN AS: +# +# ORACLE_HOME=/nfs/nikam/sw/oracle-10.2/ora_home_10.2/ LC_ALL=C AESOP_PASSWD= perl oracle_dump +# + +use strict; +use warnings; +use utf8; + +use lib '/nfs/nikam/sw/oracle-10.2/perl/lib/perl'; +use DBI; +use DBD::Oracle qw(:ora_types); +$ENV{'NLS_LANG'}='CZECH.UTF8'; + + +my $pass = $ENV{'AESOP_PASSWD'} // die "You must set AESOP_PASSWD\n"; +my $dbh = DBI->connect('dbi:Oracle:host=mffout.karlov.mff.cuni.cz;service_name=mffout.karlov.mff.cuni.cz;port=1521', + 'mamoper', $pass, {ora_connect_with_default_signals => [ 'INT' ], 'LongReadLen' => 1000000 }) or die; + +$, = "\t"; +$\ = "\n"; +my $dump_directory = "oracle_dks_dump"; +mkdir $dump_directory unless -d $dump_directory; + +sub dump_table($){ + my $name = shift; + print $name; + my $filename = $dump_directory."/".$name.".csv"; + open(my $fh, ">:encoding(UTF-8)", $filename); + select $fh; + my $query = $dbh->prepare("SELECT * FROM $name"); + $query->execute; + print @{$query->{NAME}}; + print @{$query->{TYPE}}; + while (my @row = $query->fetchrow_array()) { + $_//='' for @row; + s/\t/\\t/g for @row; + s/\n/\\n/g for @row; + print @row; + } + select STDOUT; + close $fh; +} + +# my $query = $dbh->prepare("SELECT table_name, TABLESPACE_NAME FROM all_tables WHERE owner='MAMOPER'"); +my $query = $dbh->prepare("SELECT table_name, owner FROM all_tables"); +$query->execute; +while (my @row = $query->fetchrow_array()) { + if (!( $row[1] eq 'SYS')) { + dump_table "$row[1].$row[0]"; + } +}