Poznamky a zdroje k Dakos exportu a importu

dakosdump/README.md

@@ -0,0 +1,136 @@
+# Poznámky k exportu a reimportu MaM dat #
+
+### Dakos Oracle dump - postup ###
+
+Na lib. stroji na KAMu spustit:
+
+    ORACLE_HOME=/nfs/nikam/sw/oracle-10.2/ora_home_10.2/ LC_ALL=C AESOP_PASSWD=<password-of-MAMOPER> perl oracle_dump
+
+Dumpne vsechny tabulky krom SYS.*
+
+    from seminar.dksdump.helpers import read_all_tables, matchrows, onerow
+    ts = read_all_tables('seminar/dksdump/oracle_dks_dump/')
+
+Načte a vypíše všechny tabulky. Yay!
+Pozor! V základu mají všechny podivný 2. řádek s bordelem! (čísla 3, 12, ...)
+
+Import do SQLite3 pomocí `.mode tabs` a `.import DKSROOT.V_SKOLY.csv DKSROOT_V_SKOLY`.
+
+### Postup pomocí sqlite3 s Dakos daty ###
+
+`MM_RESENI`, `MM_RESENI#BODY`:
+
+
+
+
+
+### Konkrétní programy pro data (starší metoda) ###
+
+Používají ovvpfile pro načtení souborů.
+
+#### DAKOS ID pro nektere skoly ####
+
+    for i in sk[1:]:
+	f = Skola.objects.filter(nazev=i['NAZEV'], izo=i['IZO']);
+	if len(f)==1:
+	    f[0].dakos_id = i['ID']; f[0].save()
+
+
+#### Resitel ####
+
+    for i in ts['BW-resitelia-export']:
+	Resitel.objects.create(jmeno=i['jmeno'], prijmeni=i['prijmeni'], pohlavi_muz=(i['pohlavi']==u'mu\u017e'),
+	    rok_maturity=int(i['rok_maturity']), email=i['email'], telefon=i['telefon'], zasilat=Resitel.ZASILAT_DOMU,
+	    ulice=i['ulica']+' '+i['cislo'], mesto=i['obec'], psc=i['psc'], stat=i['stat'], dakos_id=i['rid'],
+	    poznamka="{'skola': '%s', 'dakos_skola_id': '%s'}\n" % (i['skola'], i['dakos_skola_id'], ))
+
+Doplneni nekterych skol:
+
+    ae = ts['Aesop-MaM-export-2014-10-20-mod']
+    skoly = ts['Aesop-skoly-export-2015-05-14-mod']
+    for i in ae:
+	r = Resitel.objects.get(jmeno=i['name'], prijmeni=i['surname']);
+	izo = i['school'].lstrip('izo:');
+	aesop_id = [s for s in skoly if izo in repr(s)][0]['id-aesop'];
+	sch = Skola.objects.get(aesop_id='aesop:'+aesop_id);
+	r.skola=sch; r.save()
+
+#### Cisla ####
+
+Hlavne ze souboru `archiv/seznam.csv`
+Navic nejaka data z DB, ale nic moc
+
+    for i in ts['BW-archiv-cisla-copy-mod']: Cislo.objects.create(rocnik=Rocnik.objects.get(rocnik=int(i['rocnik'])), cislo=i['cislo'])
+
+#### Uzivatele ####
+
+    for i in ts['MAMOPER.MM_PASSWD']:
+	if not User.objects.filter(username=i['LOGIN']): User.objects.create_user(i['LOGIN'])
+
+#### Problemy ####
+
+`MM_ZADANIA`:
+
+    typtable={'1': Problem.TYP_ULOHA, '2':Problem.TYP_TEMA, '3':Problem.TYP_SERIAL}
+    for i in ts['MAMOPER.MM_ZADANIA']:
+	print i
+	c = Cislo.objects.get(cislo=i['CISLO'] if i['CISLO']!='9' else '1', rocnik__rocnik=int(i['ROCNIK']))
+	opravovatel = User.objects.get(username=i['OPRAVUJE']) if i['OPRAVUJE'] else None
+	Problem.objects.create(nazev=i['NAZOV'], typ=typtable[i['TYP']], stav=Problem.STAV_ZADANY, opravovatel=opravovatel,
+	    kod=i['ULOHA'], cislo_zadani=c, body=int(i['MAX_BODY']) if i['MAX_BODY'] else None,
+	    dakos_id='ZAD:%s.%s.%s.%s'%(i['ROCNIK'], i['CISLO'], i['ULOHA'], i['TYP'],))
+
+`MM_DOZ`:
+Dekodovani stavu navrhu (MFIO - oblast, Tema, Konfera, Pouzita, Zamitnuta, Doplnit (?))
+
+    def dectag(x): return ["MFIOTKPZD"[bi] for bi in range(0, 9) if (int(x) & (1 << bi))]
+    from django.utils.html import escape
+    def transdate(s):
+	if '.' not in s: return None
+	d,m,r = s.split('.'); return datetime.date(int(r)+2000, int(m), int(d))
+    for i in ts['MAMOPER.MM_DOZ']:
+	print i;
+	tags=dectag(i['ZARADENIE']);
+	autor = User.objects.get(username=i['ORG']) if i['ORG'] else None;
+	Problem.objects.create(nazev=i['NAZOV'], typ=Problem.TYP_TEMA if 'T' in tags else Problem.TYP_ULOHA,
+	    stav=Problem.STAV_ZADANY if 'P' in tags else (Problem.STAV_SMAZANY if 'Z' in tags else Problem.STAV_NAVRH),
+	    dakos_id='DOZ:%s'%(i['ID'], ), text_problemu_org=escape(i['ZADANIE']), timestamp=transdate(i['CAS']))
+
+`MM_AZAD`:
+
+    def romdec(rom):
+	if not rom: return 0
+	if rom[0] == 'X': return 10 + romdec(rom[1:])
+	if rom[-1] == 'X': return 10 - romdec(rom[:-1])
+	if rom[0] == 'V': return 5 + romdec(rom[1:])
+	if rom[-1] == 'V': return 5 - romdec(rom[:-1])
+	if rom[0] == 'I': return 1 + romdec(rom[1:])
+	assert False
+    for i in ts['MAMOPER.MM_AZAD'][1:]:
+	print i;
+	tags=dectag(i['ZAMERANIE']);
+	r = romdec(i['ROC_Z']);
+	c1 = Cislo.objects.filter(rocnik__rocnik=r, cislo=i['CIS_Z']);
+	c1 = c1[0] if len(c1) else None;
+	c2 = Cislo.objects.filter(rocnik__rocnik=r, cislo=i['CIS_V']);
+	c2 = c2[0] if len(c2) else None;
+	Problem.objects.create(nazev=i['NAZOV'], typ=Problem.TYP_TEMA if 'T' in tags else Problem.TYP_ULOHA, stav=Problem.STAV_ZADANY,
+	    kod=i['ZADANIE'][1:], cislo_zadani=c1, cislo_reseni=c2, dakos_id='AZAD:%s'%(i['ID'], ),
+	    text_problemu_org=escape(u'Abstrakt:\n'+i['ABSTRAKT']))
+    for i in ts['MAMOPER.MM_AZAD'][1:]:
+	print i; tags=dectag(i['ZAMERANIE']); p = Problem.objects.get(dakos_id='AZAD:%s'%(i['ID'], ));
+	[p.zamereni.add(t) for t in tags if t in 'MFIO']; p.save()
+
+#### Reseni ####
+
+`MM_RIESENIA`:
+
+    for i in ts['MAMOPER.MM_RIESENIA']:
+	tstp = datetime.datetime.fromtimestamp(int(i['DORUCENA']));
+	print i, tstp;
+	r = Resitel.objects.get(dakos_id=i['RIESITEL']);
+	u = Problem.objects.get(dakos_id="ZAD:%s.%s.%s.%s" % (i['ROCNIK'], i['CISLO'], i['ULOHA'], i['TYP'],));
+	Reseni.objects.create(problem=u, resitel=r, body=int(i['POCET_BODOV'] or '0'),
+	    poznamka='Zdroj: MM_RIESENIA\nForma: %s\n%s' % ({'1':'E', '2':'P'}[i['FORMA']], i['POZNAMKA'], ), timestamp=tstp)
+
+

dakosdump/oracle_dump

@@ -0,0 +1,54 @@
+#!/usr/bin/perl
+
+# RUN AS:
+#
+# ORACLE_HOME=/nfs/nikam/sw/oracle-10.2/ora_home_10.2/ LC_ALL=C AESOP_PASSWD=<password-of-MAMOPER> perl oracle_dump
+#
+
+use strict;
+use warnings;
+use utf8;
+
+use lib '/nfs/nikam/sw/oracle-10.2/perl/lib/perl';
+use DBI;
+use DBD::Oracle qw(:ora_types);
+$ENV{'NLS_LANG'}='CZECH.UTF8';
+
+
+my $pass = $ENV{'AESOP_PASSWD'} // die "You must set AESOP_PASSWD\n";
+my $dbh = DBI->connect('dbi:Oracle:host=mffout.karlov.mff.cuni.cz;service_name=mffout.karlov.mff.cuni.cz;port=1521',
+  'mamoper', $pass, {ora_connect_with_default_signals => [ 'INT' ], 'LongReadLen' => 1000000 }) or die;
+
+$, = "\t";
+$\ = "\n";
+my $dump_directory = "oracle_dks_dump";
+mkdir $dump_directory unless -d $dump_directory;
+
+sub dump_table($){
+	my $name = shift;
+	print $name;
+	my $filename = $dump_directory."/".$name.".csv";
+	open(my $fh, ">:encoding(UTF-8)", $filename);
+	select $fh;
+	my $query = $dbh->prepare("SELECT * FROM $name");
+	$query->execute;
+	print @{$query->{NAME}};
+	print @{$query->{TYPE}};
+	while (my @row = $query->fetchrow_array()) {
+		$_//='' for @row;
+		s/\t/\\t/g for @row;
+		s/\n/\\n/g for @row;
+		print @row;
+	}
+	select STDOUT;
+	close $fh;
+}
+
+# my $query = $dbh->prepare("SELECT table_name, TABLESPACE_NAME FROM all_tables WHERE owner='MAMOPER'");
+my $query = $dbh->prepare("SELECT table_name, owner FROM all_tables");
+$query->execute;
+while (my @row = $query->fetchrow_array()) {
+	if (!( $row[1] eq 'SYS')) {
+		dump_table "$row[1].$row[0]"; 
+	}
+}