Browse Source

Poznamky a zdroje k Dakos exportu a importu

remotes/origin/feincms
Tomas Gavenciak 10 years ago
parent
commit
8082677655
  1. 136
      dakosdump/README.md
  2. 0
      dakosdump/__init__.py
  3. 0
      dakosdump/helpers.py
  4. 54
      dakosdump/oracle_dump

136
dakosdump/README.md

@ -0,0 +1,136 @@
# Poznámky k exportu a reimportu MaM dat #
### Dakos Oracle dump - postup ###
Na lib. stroji na KAMu spustit:
ORACLE_HOME=/nfs/nikam/sw/oracle-10.2/ora_home_10.2/ LC_ALL=C AESOP_PASSWD=<password-of-MAMOPER> perl oracle_dump
Dumpne vsechny tabulky krom SYS.*
from seminar.dksdump.helpers import read_all_tables, matchrows, onerow
ts = read_all_tables('seminar/dksdump/oracle_dks_dump/')
Načte a vypíše všechny tabulky. Yay!
Pozor! V základu mají všechny podivný 2. řádek s bordelem! (čísla 3, 12, ...)
Import do SQLite3 pomocí `.mode tabs` a `.import DKSROOT.V_SKOLY.csv DKSROOT_V_SKOLY`.
### Postup pomocí sqlite3 s Dakos daty ###
`MM_RESENI`, `MM_RESENI#BODY`:
### Konkrétní programy pro data (starší metoda) ###
Používají ovvpfile pro načtení souborů.
#### DAKOS ID pro nektere skoly ####
for i in sk[1:]:
f = Skola.objects.filter(nazev=i['NAZEV'], izo=i['IZO']);
if len(f)==1:
f[0].dakos_id = i['ID']; f[0].save()
#### Resitel ####
for i in ts['BW-resitelia-export']:
Resitel.objects.create(jmeno=i['jmeno'], prijmeni=i['prijmeni'], pohlavi_muz=(i['pohlavi']==u'mu\u017e'),
rok_maturity=int(i['rok_maturity']), email=i['email'], telefon=i['telefon'], zasilat=Resitel.ZASILAT_DOMU,
ulice=i['ulica']+' '+i['cislo'], mesto=i['obec'], psc=i['psc'], stat=i['stat'], dakos_id=i['rid'],
poznamka="{'skola': '%s', 'dakos_skola_id': '%s'}\n" % (i['skola'], i['dakos_skola_id'], ))
Doplneni nekterych skol:
ae = ts['Aesop-MaM-export-2014-10-20-mod']
skoly = ts['Aesop-skoly-export-2015-05-14-mod']
for i in ae:
r = Resitel.objects.get(jmeno=i['name'], prijmeni=i['surname']);
izo = i['school'].lstrip('izo:');
aesop_id = [s for s in skoly if izo in repr(s)][0]['id-aesop'];
sch = Skola.objects.get(aesop_id='aesop:'+aesop_id);
r.skola=sch; r.save()
#### Cisla ####
Hlavne ze souboru `archiv/seznam.csv`
Navic nejaka data z DB, ale nic moc
for i in ts['BW-archiv-cisla-copy-mod']: Cislo.objects.create(rocnik=Rocnik.objects.get(rocnik=int(i['rocnik'])), cislo=i['cislo'])
#### Uzivatele ####
for i in ts['MAMOPER.MM_PASSWD']:
if not User.objects.filter(username=i['LOGIN']): User.objects.create_user(i['LOGIN'])
#### Problemy ####
`MM_ZADANIA`:
typtable={'1': Problem.TYP_ULOHA, '2':Problem.TYP_TEMA, '3':Problem.TYP_SERIAL}
for i in ts['MAMOPER.MM_ZADANIA']:
print i
c = Cislo.objects.get(cislo=i['CISLO'] if i['CISLO']!='9' else '1', rocnik__rocnik=int(i['ROCNIK']))
opravovatel = User.objects.get(username=i['OPRAVUJE']) if i['OPRAVUJE'] else None
Problem.objects.create(nazev=i['NAZOV'], typ=typtable[i['TYP']], stav=Problem.STAV_ZADANY, opravovatel=opravovatel,
kod=i['ULOHA'], cislo_zadani=c, body=int(i['MAX_BODY']) if i['MAX_BODY'] else None,
dakos_id='ZAD:%s.%s.%s.%s'%(i['ROCNIK'], i['CISLO'], i['ULOHA'], i['TYP'],))
`MM_DOZ`:
Dekodovani stavu navrhu (MFIO - oblast, Tema, Konfera, Pouzita, Zamitnuta, Doplnit (?))
def dectag(x): return ["MFIOTKPZD"[bi] for bi in range(0, 9) if (int(x) & (1 << bi))]
from django.utils.html import escape
def transdate(s):
if '.' not in s: return None
d,m,r = s.split('.'); return datetime.date(int(r)+2000, int(m), int(d))
for i in ts['MAMOPER.MM_DOZ']:
print i;
tags=dectag(i['ZARADENIE']);
autor = User.objects.get(username=i['ORG']) if i['ORG'] else None;
Problem.objects.create(nazev=i['NAZOV'], typ=Problem.TYP_TEMA if 'T' in tags else Problem.TYP_ULOHA,
stav=Problem.STAV_ZADANY if 'P' in tags else (Problem.STAV_SMAZANY if 'Z' in tags else Problem.STAV_NAVRH),
dakos_id='DOZ:%s'%(i['ID'], ), text_problemu_org=escape(i['ZADANIE']), timestamp=transdate(i['CAS']))
`MM_AZAD`:
def romdec(rom):
if not rom: return 0
if rom[0] == 'X': return 10 + romdec(rom[1:])
if rom[-1] == 'X': return 10 - romdec(rom[:-1])
if rom[0] == 'V': return 5 + romdec(rom[1:])
if rom[-1] == 'V': return 5 - romdec(rom[:-1])
if rom[0] == 'I': return 1 + romdec(rom[1:])
assert False
for i in ts['MAMOPER.MM_AZAD'][1:]:
print i;
tags=dectag(i['ZAMERANIE']);
r = romdec(i['ROC_Z']);
c1 = Cislo.objects.filter(rocnik__rocnik=r, cislo=i['CIS_Z']);
c1 = c1[0] if len(c1) else None;
c2 = Cislo.objects.filter(rocnik__rocnik=r, cislo=i['CIS_V']);
c2 = c2[0] if len(c2) else None;
Problem.objects.create(nazev=i['NAZOV'], typ=Problem.TYP_TEMA if 'T' in tags else Problem.TYP_ULOHA, stav=Problem.STAV_ZADANY,
kod=i['ZADANIE'][1:], cislo_zadani=c1, cislo_reseni=c2, dakos_id='AZAD:%s'%(i['ID'], ),
text_problemu_org=escape(u'Abstrakt:\n'+i['ABSTRAKT']))
for i in ts['MAMOPER.MM_AZAD'][1:]:
print i; tags=dectag(i['ZAMERANIE']); p = Problem.objects.get(dakos_id='AZAD:%s'%(i['ID'], ));
[p.zamereni.add(t) for t in tags if t in 'MFIO']; p.save()
#### Reseni ####
`MM_RIESENIA`:
for i in ts['MAMOPER.MM_RIESENIA']:
tstp = datetime.datetime.fromtimestamp(int(i['DORUCENA']));
print i, tstp;
r = Resitel.objects.get(dakos_id=i['RIESITEL']);
u = Problem.objects.get(dakos_id="ZAD:%s.%s.%s.%s" % (i['ROCNIK'], i['CISLO'], i['ULOHA'], i['TYP'],));
Reseni.objects.create(problem=u, resitel=r, body=int(i['POCET_BODOV'] or '0'),
poznamka='Zdroj: MM_RIESENIA\nForma: %s\n%s' % ({'1':'E', '2':'P'}[i['FORMA']], i['POZNAMKA'], ), timestamp=tstp)

0
dakosdump/__init__.py

0
seminar/dksdump/helpers.py → dakosdump/helpers.py

54
dakosdump/oracle_dump

@ -0,0 +1,54 @@
#!/usr/bin/perl
# RUN AS:
#
# ORACLE_HOME=/nfs/nikam/sw/oracle-10.2/ora_home_10.2/ LC_ALL=C AESOP_PASSWD=<password-of-MAMOPER> perl oracle_dump
#
use strict;
use warnings;
use utf8;
use lib '/nfs/nikam/sw/oracle-10.2/perl/lib/perl';
use DBI;
use DBD::Oracle qw(:ora_types);
$ENV{'NLS_LANG'}='CZECH.UTF8';
my $pass = $ENV{'AESOP_PASSWD'} // die "You must set AESOP_PASSWD\n";
my $dbh = DBI->connect('dbi:Oracle:host=mffout.karlov.mff.cuni.cz;service_name=mffout.karlov.mff.cuni.cz;port=1521',
'mamoper', $pass, {ora_connect_with_default_signals => [ 'INT' ], 'LongReadLen' => 1000000 }) or die;
$, = "\t";
$\ = "\n";
my $dump_directory = "oracle_dks_dump";
mkdir $dump_directory unless -d $dump_directory;
sub dump_table($){
my $name = shift;
print $name;
my $filename = $dump_directory."/".$name.".csv";
open(my $fh, ">:encoding(UTF-8)", $filename);
select $fh;
my $query = $dbh->prepare("SELECT * FROM $name");
$query->execute;
print @{$query->{NAME}};
print @{$query->{TYPE}};
while (my @row = $query->fetchrow_array()) {
$_//='' for @row;
s/\t/\\t/g for @row;
s/\n/\\n/g for @row;
print @row;
}
select STDOUT;
close $fh;
}
# my $query = $dbh->prepare("SELECT table_name, TABLESPACE_NAME FROM all_tables WHERE owner='MAMOPER'");
my $query = $dbh->prepare("SELECT table_name, owner FROM all_tables");
$query->execute;
while (my @row = $query->fetchrow_array()) {
if (!( $row[1] eq 'SYS')) {
dump_table "$row[1].$row[0]";
}
}
Loading…
Cancel
Save