[Koha-cvs] koha/misc/migration_tools rebuild_zebra.pl [rel_3_0]

[paul poulain]

CVSROOT:        /sources/koha
Module name:    koha
Branch:         rel_3_0
Changes by:     paul poulain <tipaul>   07/01/24 13:59:35

Modified files:
        misc/migration_tools: rebuild_zebra.pl 

Log message:
        improving rebuild_zebra.pl
        this script now works fine and can create all necessary zebra dirs, 
export biblios/authorities & reindex everything in zebra

CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/koha/misc/migration_tools/rebuild_zebra.pl?cvsroot=koha&only_with_tag=rel_3_0&r1=1.4.2.12&r2=1.4.2.13

Patches:
Index: rebuild_zebra.pl
===================================================================
RCS file: /sources/koha/koha/misc/migration_tools/rebuild_zebra.pl,v
retrieving revision 1.4.2.12
retrieving revision 1.4.2.13
diff -u -b -r1.4.2.12 -r1.4.2.13
--- rebuild_zebra.pl    6 Dec 2006 21:55:38 -0000       1.4.2.12
+++ rebuild_zebra.pl    24 Jan 2007 13:59:35 -0000      1.4.2.13
@@ -14,16 +14,20 @@
 $|=1; # flushes output
 
 # limit for database dumping
-my $limit;# = "LIMIT 100";
+my $limit;# = "LIMIT 500";
 my $directory;
 my $skip_export;
 my $keep_export;
 my $reset;
+my $biblios;
+my $authorities;
 GetOptions(
        'd:s'      => \$directory,
        'reset'      => \$reset,
        's'        => \$skip_export,
        'k'        => \$keep_export,
+       'b'        => \$biblios,
+       'a'        => \$authorities,
        );
 
 $directory = "export" unless $directory;
@@ -124,51 +128,52 @@
 # AUTHORITIES creating directory structure
 #
 my $created_dir_or_file = 0;
-print "====================\n";
-print "checking directories & files for authorities\n";
-print "====================\n";
-unless (-d "$authorityserverdir") {
+if ($authorities) {
+    print "====================\n";
+    print "checking directories & files for authorities\n";
+    print "====================\n";
+    unless (-d "$authorityserverdir") {
     system("mkdir -p $authorityserverdir");
     print "Info: created $authorityserverdir\n";
     $created_dir_or_file++;
-}
-unless (-d "$authorityserverdir/lock") {
+    }
+    unless (-d "$authorityserverdir/lock") {
     mkdir "$authorityserverdir/lock";
     print "Info: created $authorityserverdir/lock\n";
     $created_dir_or_file++;
-}
-unless (-d "$authorityserverdir/register") {
+    }
+    unless (-d "$authorityserverdir/register") {
     mkdir "$authorityserverdir/register";
     print "Info: created $authorityserverdir/register\n";
     $created_dir_or_file++;
-}
-unless (-d "$authorityserverdir/shadow") {
+    }
+    unless (-d "$authorityserverdir/shadow") {
     mkdir "$authorityserverdir/shadow";
     print "Info: created $authorityserverdir/shadow\n";
     $created_dir_or_file++;
-}
-unless (-d "$authorityserverdir/tab") {
+    }
+    unless (-d "$authorityserverdir/tab") {
     mkdir "$authorityserverdir/tab";
     print "Info: created $authorityserverdir/tab\n";
     $created_dir_or_file++;
-}
-unless (-d "$authorityserverdir/key") {
+    }
+    unless (-d "$authorityserverdir/key") {
     mkdir "$authorityserverdir/key";
     print "Info: created $authorityserverdir/key\n";
     $created_dir_or_file++;
-}
+    }
 
-unless (-d "$authorityserverdir/etc") {
+    unless (-d "$authorityserverdir/etc") {
     mkdir "$authorityserverdir/etc";
     print "Info: created $authorityserverdir/etc\n";
     $created_dir_or_file++;
-}
+    }
 
-#
-# AUTHORITIES : copying mandatory files
-#
-# the record model, depending on marc flavour
-unless (-f "$authorityserverdir/tab/record.abs") {
+    #
+    # AUTHORITIES : copying mandatory files
+    #
+    # the record model, depending on marc flavour
+    unless (-f "$authorityserverdir/tab/record.abs") {
     if (C4::Context->preference("marcflavour") eq "UNIMARC") {
         system("cp -f $kohadir/misc/zebra/record_authorities_unimarc.abs 
$authorityserverdir/tab/record.abs");
         print "Info: copied record.abs for UNIMARC\n";
@@ -177,102 +182,101 @@
         print "Info: copied record.abs for USMARC\n";
     }
     $created_dir_or_file++;
-}
-unless (-f "$authorityserverdir/tab/sort-string-utf_french.chr") {
+    }
+    unless (-f "$authorityserverdir/tab/sort-string-utf_french.chr") {
     system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$authorityserverdir/tab/sort-string-utf.chr");
     print "Info: copied sort-string-utf.chr\n";
     $created_dir_or_file++;
-}
-unless (-f "$authorityserverdir/tab/word-phrase-utf.chr") {
+    }
+    unless (-f "$authorityserverdir/tab/word-phrase-utf.chr") {
     system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$authorityserverdir/tab/word-phrase-utf.chr");
     print "Info: copied word-phase-utf.chr\n";
     $created_dir_or_file++;
-}
-unless (-f "$authorityserverdir/tab/auth1.att") {
+    }
+    unless (-f "$authorityserverdir/tab/auth1.att") {
     system("cp -f $kohadir/misc/zebra/bib1_authorities.att 
$authorityserverdir/tab/auth1.att");
     print "Info: copied auth1.att\n";
     $created_dir_or_file++;
-}
-unless (-f "$authorityserverdir/tab/default.idx") {
+    }
+    unless (-f "$authorityserverdir/tab/default.idx") {
     system("cp -f $kohadir/misc/zebra/default.idx 
$authorityserverdir/tab/default.idx");
     print "Info: copied default.idx\n";
     $created_dir_or_file++;
-}
+    }
 
-unless (-f C4::Context->zebraoptions('authorityserver')->{ccl2rpn}) {
+    unless (-f C4::Context->zebraoptions('authorityserver')->{ccl2rpn}) {
      system("cp -f $kohadir/misc/zebra/ccl.properties 
".C4::Context->zebraoptions('authorityserver')->{ccl2rpn});
-#    system("cp -f $kohadir/misc/zebra/ccl.properties 
$authorityserverdir/etc/ccl.properties");
+    #    system("cp -f $kohadir/misc/zebra/ccl.properties 
$authorityserverdir/etc/ccl.properties");
     print "Info: copied ccl.properties\n";
     $created_dir_or_file++;
-}
-unless (-f C4::Context->zebraconfig('authorityserver')->{cql2rpn}) {
+    }
+    unless (-f C4::Context->zebraconfig('authorityserver')->{cql2rpn}) {
     system("cp -f $kohadir/misc/zebra/pqf.properties 
".C4::Context->zebraconfig('authorityserver')->{cql2rpn});
     print "Info: copied pqf.properties\n";
     $created_dir_or_file++;
-}
+    }
 
-#
-# AUTHORITIES : copying mandatory files
-#
-unless (-f C4::Context->zebraconfig('authorityserver')->{config}) {
-open ZD,">:utf8 ",C4::Context->zebraconfig('authorityserver')->{config};
-print ZD "
-# generated by KOHA/misc/migration_tools/rebuild_zebra.pl 
-profilePath:\${srcdir:-.}:$authorityserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
-
-encoding: UTF-8
-# Files that describe the attribute sets supported.
-attset: auth1.att
-attset: explain.att
-attset: gils.att
-
-modulePath:$modulesdir/modules/
-systag sysno rank
-# Specify record type
-iso2709.recordType:grs.marcxml.record
-recordType:grs.xml
-recordId: (auth1,Local-Number)
-storeKeys:1
-storeData:1
-
-
-# Lock File Area
-lockDir: $authorityserverdir/lock
-perm.anonymous:r
-perm.kohaadmin:rw
-passw.kohalis
-shadow
-register: $authorityserverdir/register:4G
-shadow: $authorityserverdir/shadow:4G
-
-# Temp File area for result sets
-setTmpDir: $authorityserverdir/tmp
-
-# Temp File area for index program
-keyTmpDir: $authorityserverdir/key
-
-# Approx. Memory usage during indexing
-memMax: 40M
-rank:rank-1
-";
+    #
+    # AUTHORITIES : copying mandatory files
+    #
+    unless (-f C4::Context->zebraconfig('authorityserver')->{config}) {
+    open ZD,">:utf8 ",C4::Context->zebraconfig('authorityserver')->{config};
+    print ZD "
+    # generated by KOHA/misc/migration_tools/rebuild_zebra.pl 
+    
profilePath:\${srcdir:-.}:$authorityserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
+    
+    encoding: UTF-8
+    # Files that describe the attribute sets supported.
+    attset: auth1.att
+    attset: explain.att
+    attset: gils.att
+    
+    modulePath:$modulesdir/modules/
+    # Specify record type
+    iso2709.recordType:grs.marcxml.record
+    recordType:grs.xml
+    recordId: (auth1,Local-Number)
+    storeKeys:1
+    storeData:1
+    
+    
+    # Lock File Area
+    lockDir: $authorityserverdir/lock
+    perm.anonymous:r
+    perm.kohaadmin:rw
+    passw.kohalis
+    shadow
+    register: $authorityserverdir/register:4G
+    shadow: $authorityserverdir/shadow:4G
+    
+    # Temp File area for result sets
+    setTmpDir: $authorityserverdir/tmp
+    
+    # Temp File area for index program
+    keyTmpDir: $authorityserverdir/key
+    
+    # Approx. Memory usage during indexing
+    memMax: 40M
+    rank:rank-1
+    ";
     print "Info: creating zebra-authorities.cfg\n";
     $created_dir_or_file++;
-}
+    }
 
-if ($created_dir_or_file) {
+    if ($created_dir_or_file) {
     print "Info: created : $created_dir_or_file directories & files\n";
-} else {
+    } else {
     print "Info: file & directories OK\n";
-}
+    }
 
-#
-# exporting authorities
-#
-if ($skip_export) {
+    #
+    # exporting authorities
+    #
+    if ($skip_export) {
     print "====================\n";
     print "SKIPPING authorities export\n";
     print "====================\n";
-} else {
+    } else {
     print "====================\n";
     print "exporting authorities\n";
     print "====================\n";
@@ -291,70 +295,73 @@
         print OUT $record->as_usmarc();
     }
     close(OUT);
-}
-
-#
-# and reindexing everything
-#
-print "====================\n";
-print "REINDEXING zebra\n";
-print "====================\n";
-system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('authorityserver')->{config}." -d authorities init") 
if ($reset);
-system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('authorityserver')->{config}." -d authorities update 
$directory/authorities");
-system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('authorityserver')->{config}." -d authorities 
commit");
+    }
 
+    #
+    # and reindexing everything
+    #
+    print "====================\n";
+    print "REINDEXING zebra\n";
+    print "====================\n";
+    system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('authorityserver')->{config}." -d authorities init") 
if ($reset);
+    system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('authorityserver')->{config}." -d authorities update 
$directory/authorities");
+    system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('authorityserver')->{config}." -d authorities 
commit");
+} else {
+    print "skipping authorities\n";
+}
 
#################################################################################################################
 #                        BIBLIOS 
 
#################################################################################################################
 
-print "====================\n";
-print "checking directories & files for biblios\n";
-print "====================\n";
+if ($biblios) {
+    print "====================\n";
+    print "checking directories & files for biblios\n";
+    print "====================\n";
 
-#
-# BIBLIOS : creating directory structure
-#
-unless (-d "$biblioserverdir") {
+    #
+    # BIBLIOS : creating directory structure
+    #
+    unless (-d "$biblioserverdir") {
     system("mkdir -p $biblioserverdir");
     print "Info: created $biblioserverdir\n";
     $created_dir_or_file++;
-}
-unless (-d "$biblioserverdir/lock") {
+    }
+    unless (-d "$biblioserverdir/lock") {
     mkdir "$biblioserverdir/lock";
     print "Info: created $biblioserverdir/lock\n";
     $created_dir_or_file++;
-}
-unless (-d "$biblioserverdir/register") {
+    }
+    unless (-d "$biblioserverdir/register") {
     mkdir "$biblioserverdir/register";
     print "Info: created $biblioserverdir/register\n";
     $created_dir_or_file++;
-}
-unless (-d "$biblioserverdir/shadow") {
+    }
+    unless (-d "$biblioserverdir/shadow") {
     mkdir "$biblioserverdir/shadow";
     print "Info: created $biblioserverdir/shadow\n";
     $created_dir_or_file++;
-}
-unless (-d "$biblioserverdir/tab") {
+    }
+    unless (-d "$biblioserverdir/tab") {
     mkdir "$biblioserverdir/tab";
     print "Info: created $biblioserverdir/tab\n";
     $created_dir_or_file++;
-}
-unless (-d "$biblioserverdir/key") {
+    }
+    unless (-d "$biblioserverdir/key") {
     mkdir "$biblioserverdir/key";
     print "Info: created $biblioserverdir/key\n";
     $created_dir_or_file++;
-}
-unless (-d "$biblioserverdir/etc") {
+    }
+    unless (-d "$biblioserverdir/etc") {
     mkdir "$biblioserverdir/etc";
     print "Info: created $biblioserverdir/etc\n";
     $created_dir_or_file++;
-}
+    }
 
-#
-# BIBLIOS : copying mandatory files
-#
-# the record model, depending on marc flavour
-unless (-f "$biblioserverdir/tab/record.abs") {
+    #
+    # BIBLIOS : copying mandatory files
+    #
+    # the record model, depending on marc flavour
+    unless (-f "$biblioserverdir/tab/record.abs") {
     if (C4::Context->preference("marcflavour") eq "UNIMARC") {
         system("cp -f $kohadir/misc/zebra/record_biblios_unimarc.abs 
$biblioserverdir/tab/record.abs");
         print "Info: copied record.abs for UNIMARC\n";
@@ -363,102 +370,101 @@
         print "Info: copied record.abs for USMARC\n";
     }
     $created_dir_or_file++;
-}
-unless (-f "$biblioserverdir/tab/sort-string-utf_french.chr") {
+    }
+    unless (-f "$biblioserverdir/tab/sort-string-utf_french.chr") {
     system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$biblioserverdir/tab/sort-string-utf.chr");
     print "Info: copied sort-string-utf.chr\n";
     $created_dir_or_file++;
-}
-unless (-f "$biblioserverdir/tab/word-phrase-utf.chr") {
+    }
+    unless (-f "$biblioserverdir/tab/word-phrase-utf.chr") {
     system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$biblioserverdir/tab/word-phrase-utf.chr");
     print "Info: copied word-phase-utf.chr\n";
     $created_dir_or_file++;
-}
-unless (-f "$biblioserverdir/tab/bib1.att") {
+    }
+    unless (-f "$biblioserverdir/tab/bib1.att") {
     system("cp -f $kohadir/misc/zebra/bib1_biblios.att 
$biblioserverdir/tab/bib1.att");
     print "Info: copied bib1.att\n";
     $created_dir_or_file++;
-}
-unless (-f "$biblioserverdir/tab/default.idx") {
+    }
+    unless (-f "$biblioserverdir/tab/default.idx") {
     system("cp -f $kohadir/misc/zebra/default.idx 
$biblioserverdir/tab/default.idx");
     print "Info: copied default.idx\n";
     $created_dir_or_file++;
-}
-unless (-f C4::Context->zebraoptions('biblioserver')->{ccl2rpn}) {
+    }
+    unless (-f C4::Context->zebraoptions('biblioserver')->{ccl2rpn}) {
     system("cp -f $kohadir/misc/zebra/ccl.properties 
".C4::Context->zebraoptions('biblioserver')->{ccl2rpn});
-#    system("cp -f $kohadir/misc/zebra/ccl.properties 
$biblioserverdir/etc/ccl.properties");
+    #    system("cp -f $kohadir/misc/zebra/ccl.properties 
$biblioserverdir/etc/ccl.properties");
     print "Info: copied ccl.properties\n";
     $created_dir_or_file++;
-}
-unless (-f C4::Context->zebraconfig('biblioserver')->{cql2rpn}) {
+    }
+    unless (-f C4::Context->zebraconfig('biblioserver')->{cql2rpn}) {
     system("cp -f $kohadir/misc/zebra/pqf.properties 
".C4::Context->zebraconfig('authorityserver')->{cql2rpn});
     print "Info: copied pqf.properties\n";
     $created_dir_or_file++;
-}
+    }
 
-#
-# BIBLIOS : copying mandatory files
-#
-unless (-f C4::Context->zebraconfig('biblioserver')->{config}) {
-open ZD,">:utf8 ",C4::Context->zebraconfig('biblioserver')->{config};
-print ZD "
-# generated by KOHA/misc/migrtion_tools/rebuild_zebra.pl 
-profilePath:\${srcdir:-.}:$biblioserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
-
-encoding: UTF-8
-# Files that describe the attribute sets supported.
-attset:bib1.att
-attset:explain.att
-attset:gils.att
-
-modulePath:$modulesdir/modules/
-systag sysno rank
-# Specify record type
-iso2709.recordType:grs.marcxml.record
-recordType:grs.xml
-recordId: (bib1,Local-Number)
-storeKeys:1
-storeData:1
-
-
-# Lock File Area
-lockDir: $biblioserverdir/lock
-perm.anonymous:r
-perm.kohaadmin:rw
-passw.kohalis
-shadow
-register: $biblioserverdir/register:4G
-shadow: $biblioserverdir/shadow:4G
-
-# Temp File area for result sets
-setTmpDir: $biblioserverdir/tmp
-
-# Temp File area for index program
-keyTmpDir: $biblioserverdir/key
-
-# Approx. Memory usage during indexing
-memMax: 40M
-rank:rank-1
-";
+    #
+    # BIBLIOS : copying mandatory files
+    #
+    unless (-f C4::Context->zebraconfig('biblioserver')->{config}) {
+    open ZD,">:utf8 ",C4::Context->zebraconfig('biblioserver')->{config};
+    print ZD "
+    # generated by KOHA/misc/migrtion_tools/rebuild_zebra.pl 
+    
profilePath:\${srcdir:-.}:$biblioserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
+    
+    encoding: UTF-8
+    # Files that describe the attribute sets supported.
+    attset:bib1.att
+    attset:explain.att
+    attset:gils.att
+    
+    modulePath:$modulesdir/modules/
+    # Specify record type
+    iso2709.recordType:grs.marcxml.record
+    recordType:grs.xml
+    recordId: (bib1,Local-Number)
+    storeKeys:1
+    storeData:1
+    
+    
+    # Lock File Area
+    lockDir: $biblioserverdir/lock
+    perm.anonymous:r
+    perm.kohaadmin:rw
+    passw.kohalis
+    shadow
+    register: $biblioserverdir/register:4G
+    shadow: $biblioserverdir/shadow:4G
+    
+    # Temp File area for result sets
+    setTmpDir: $biblioserverdir/tmp
+    
+    # Temp File area for index program
+    keyTmpDir: $biblioserverdir/key
+    
+    # Approx. Memory usage during indexing
+    memMax: 40M
+    rank:rank-1
+    ";
     print "Info: creating zebra-biblios.cfg\n";
     $created_dir_or_file++;
-}
+    }
 
-if ($created_dir_or_file) {
+    if ($created_dir_or_file) {
     print "Info: created : $created_dir_or_file directories & files\n";
-} else {
+    } else {
     print "Info: file & directories OK\n";
-}
+    }
 
-# die;
-#
-# exporting biblios
-#
-if ($skip_export) {
+    # die;
+    #
+    # exporting biblios
+    #
+    if ($skip_export) {
     print "====================\n";
     print "SKIPPING biblio export\n";
     print "====================\n";
-} else {
+    } else {
     print "====================\n";
     print "exporting biblios\n";
     print "====================\n";
@@ -472,10 +478,11 @@
     my $i=0;
     while (my ($biblionumber) = $sth->fetchrow) {
         my $record = GetMarcBiblio($biblionumber);
-#         warn $record->as_formatted;
-#         print $record;
+    #         warn $record->as_formatted;
+    #         print $record;
         # check that biblionumber & biblioitemnumber are stored in the MARC 
record, otherwise, add them & update the biblioitems.marcxml data.
         my $record_correct=1;
+            next unless $record->field($biblionumbertagfield);
         if ($biblionumbertagfield eq '001') {
             unless ($record->field($biblionumbertagfield)->data()) {
                 $record_correct=0;
@@ -504,11 +511,11 @@
                     $record->append_fields($newfield);
                 }
             }
-#             warn "FIXED BIBLIONUMBER".$record->as_formatted;
+    #             warn "FIXED BIBLIONUMBER".$record->as_formatted;
         }
         unless 
($record->subfield($biblioitemnumbertagfield,$biblioitemnumbertagsubfield)) {
             $record_correct=0;
-#             warn "INCORRECT BIBLIOITEMNUMBER :".$record->as_formatted;
+    #             warn "INCORRECT BIBLIOITEMNUMBER :".$record->as_formatted;
             my $field;
             # if the field where biblionumber is already exist, just update 
it, otherwise create it
             if ($record->field($biblioitemnumbertagfield)) {
@@ -527,7 +534,7 @@
                 }
                 $record->insert_grouped_field($newfield);
             }
-#             warn "FIXED BIBLIOITEMNUMBER".$record->as_formatted;
+    #             warn "FIXED BIBLIOITEMNUMBER".$record->as_formatted;
         }
         unless ($record_correct) {
             my $update_xml = $dbh->prepare("update biblioitems set marcxml=? 
where biblionumber=?");
@@ -539,17 +546,20 @@
         print OUT $record->as_usmarc();
     }
     close(OUT);
-}
+    }
 
-#
-# and reindexing everything
-#
-print "====================\n";
-print "REINDEXING zebra\n";
-print "====================\n";
-system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios init") if 
($reset);
-system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios update 
$directory/biblios");
-system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios commit");
+    #
+    # and reindexing everything
+    #
+    print "====================\n";
+    print "REINDEXING zebra\n";
+    print "====================\n";
+    system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios init") if 
($reset);
+    system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios update 
$directory/biblios");
+    system("zebraidx -g iso2709 -c 
".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios commit");
+} else {
+    print "skipping biblios\n";
+}
 
 print "====================\n";
 print "CLEANING\n";
@@ -557,6 +567,6 @@
 if ($keep_export) {
     print "NOTHING cleaned : the $directory has been kept. You can re-run this 
script with the -s parameter if you just want to rebuild zebra after changing 
the record.abs or another zebra config file\n";
 } else {
-#     system("rm -rf $export");
-#     print "directory $export deleted\n";
+    system("rm -rf $directory");
+    print "directory $directory deleted\n";
 }