From fc9881381c8bb4d6e729532de2fe030b3ff418b3 Mon Sep 17 00:00:00 2001 From: Rick Bradshow Date: Mon, 27 Feb 2006 18:30:10 +0000 Subject: I updated all my repo scripts for the latest changes in the 0.8 version of the repo. these are still very specific to my repo, but should give a good idea of what is needed. git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1772 ce84e21b-d406-0410-9b95-82705330c041 --- tools/create-debian-pkglist.pl | 596 +++++++---------------------------------- 1 file changed, 92 insertions(+), 504 deletions(-) (limited to 'tools/create-debian-pkglist.pl') diff --git a/tools/create-debian-pkglist.pl b/tools/create-debian-pkglist.pl index 8cf6409dc..1567f5aa2 100644 --- a/tools/create-debian-pkglist.pl +++ b/tools/create-debian-pkglist.pl @@ -1,536 +1,124 @@ #!/usr/bin/perl -#this is some setup that I have to do to make everything more localized below -#get all the packages/versions from security.debian.org. this is kinda hacky since we don't locally mirror that. I would like to for speed -#issues but since we are low on disk space I am not going to bother. -#these are the files that I will need to use to get the right info. +#some quicksetup to make sure everything is in place +system('mkdir -p /disks/tmp/bcfg2-packagelists' ); -system( "wget http://security.debian.org/dists/stable/updates/main/binary-i386/Packages -O /tmp/main.Packages -q" ); -system( "wget http://security.debian.org/dists/stable/updates/contrib/binary-i386/Packages -O /tmp/contrib.Packages -q" ); -system( "wget http://security.debian.org/dists/stable/updates/non-free/binary-i386/Packages -O /tmp/nonfree.Packages -q" ); -#i now have all the files I need locally so I can do my opens properly. +#pull the correct package lists from the security sites. +#this needs to be abstracted better +system( "wget http://security.debian.org/dists/stable/updates/main/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/security-main.Packages -q" ); +system( "wget http://security.debian.org/dists/stable/updates/contrib/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/security-contrib.Packages -q" ); +system( "wget http://security.debian.org/dists/stable/updates/non-free/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/security-nonfree.Packages -q" ); -open( MAIN, "/cluster/distro/debian/dists/stable/main/binary-i386/Packages" ); -open( CONTRIB, "/cluster/distro/debian/dists/stable/contrib/binary-i386/Packages"); -open( NONFREE, "/cluster/distro/debian/dists/stable/non-free/binary-i386/Packages"); -open( NONMAIN, "/cluster/distro/debian-non-US/dists/stable/non-US/main/binary-i386/Packages" ); -open( NONCONTRIB, "/cluster/distro/debian-non-US/dists/stable/non-US/contrib/binary-i386/Packages"); -open( NONNONFREE, "/cluster/distro/debian-non-US/dists/stable/non-US/non-free/binary-i386/Packages"); -open( SECMAIN, "/tmp/main.Packages"); -open( SECCONTRIB, "/tmp/contrib.Packages"); -open( SECNONFREE, "/tmp/nonfree.Packages"); +system('cat /disks/tmp/bcfg2-packagelists/security-main.Packages /disks/tmp/bcfg2-packagelists/security-contrib.Packages /disks/tmp/bcfg2-packagelists/security-nonfree.Packages > /disks/tmp/bcfg2-packagelists/debian-stable-security.Packages'); +system('rm /disks/tmp/bcfg2-packagelists/security-main.Packages /disks/tmp/bcfg2-packagelists/security-contrib.Packages /disks/tmp/bcfg2-packagelists/security-nonfree.Packages'); -open( LOCAL, "/cluster/debian/woody/Packages"); -open( OUTFILE, ">/cluster/bcfg/images/debian-3.0/pkglist.xml" ); +#pull the correct package lists from the security sites. +#this needs to be abstracted better +system( "wget http://volatile.debian.net/debian-volatile/dists/stable/volatile/main/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/volatile-main.Packages -q" ); +system( "wget http://volatile.debian.net/debian-volatile/dists/stable/volatile/contrib/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/volatile-contrib.Packages -q" ); +system( "wget http://volatile.debian.net/debian-volatile/dists/stable/volatile/non-free/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/volatile-nonfree.Packages -q" ); -print OUTFILE "\n"; -print OUTFILE "\n"; +system('cat /disks/tmp/bcfg2-packagelists/volatile-main.Packages /disks/tmp/bcfg2-packagelists/volatile-contrib.Packages /disks/tmp/bcfg2-packagelists/volatile-nonfree.Packages > /disks/tmp/bcfg2-packagelists/debian-stable-volatile.Packages'); +system('rm /disks/tmp/bcfg2-packagelists/volatile-main.Packages /disks/tmp/bcfg2-packagelists/volatile-contrib.Packages /disks/tmp/bcfg2-packagelists/volatile-nonfree.Packages'); -@mypackages = []; +#pull the correct package lists from the security sites. +#this needs to be abstracted better +system( "wget ftp://ftp.nerim.net/debian-marillat/dists/sarge/main/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/debian-sarge-mplayer.Packages -q" ); -#get all the data from the local repo that we maintain. This is the highest precedence in package priority -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - push @mypackages, $basename; - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - print OUTFILE "\t\n" ; - $found =1; - } - } - } -} - -#now i will do all the security packages and add them to the my packages if they aren't already there. If that makes sense. -#basically i am enforceing the priority by processing the package lists inorder. - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - #print $version."\n"; - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - push @mypackages, $basename; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - #print $version."\n"; - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - push @mypackages, $basename; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - #print $version."\n"; - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - push @mypackages, $basename; - } - $known_package = 0; - $found =1; - } - } - } -} - - -#now for all the regular packages in the mirrored repos - -$known_package=0; -while( $line =
){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line =
; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} +#this is to fix local files so that my naming hack thing will playout. +system('cp /disks/debian/sarge/Packages /disks/tmp/bcfg2-packagelists/debian-sarge-local.Packages'); -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} +system('cat /disks/distro/debian/dists/sarge/main/binary-i386/Packages /disks/distro/debian/dists/sarge/contrib/binary-i386/Packages /disks/distro/debian/dists/sarge/non-free/binary-i386/Packages > /disks/tmp/bcfg2-packagelists/debian-sarge-distro.Packages'); -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} +system('cat /disks/distro/debian-non-US/dists/sarge/non-US/main/binary-i386/Packages /disks/distro/debian-non-US/dists/sarge/non-US/contrib/binary-i386/Packages /disks/distro/debian-non-US/dists/sarge/non-US/non-free/binary-i386/Packages > /disks/tmp/bcfg2-packagelists/debian-sarge-nonUS.Packages'); -print OUTFILE "\n\n"; +#this is currently still a hack, because ordering is important. +#for future refernce you must do security and then local.. then the rest.. +@files = ( + "/disks/tmp/bcfg2-packagelists/debian-stable-volatile.Packages", + "/disks/tmp/bcfg2-packagelists/debian-stable-security.Packages", + "/disks/tmp/bcfg2-packagelists/debian-sarge-local.Packages", + "/disks/tmp/bcfg2-packagelists/debian-sarge-mplayer.Packages", + "/disks/tmp/bcfg2-packagelists/debian-sarge-distro.Packages", + "/disks/tmp/bcfg2-packagelists/debian-sarge-nonUS.Packages", +); +$priority = 89; +@tmpfiles = (); -close(OUTFILE); -close(MAIN); -close(CONTRIB); -close(NONFREE); -close(NONMAIN); -close(NONCONTRIB); -close(NONNONFREE); -close(SECMAIN); -close(SECCONTRIB); -close(SECNONFREE); -close(LOCAL); +#first come the security fixes +foreach $file ( @files ){ + push( @tmpfiles, $file ); + #first we open up the imput file + open( INFILE, "$file" ) or die("could not open $file\n"); + + #then we change the name and open the output file. + $file =~ s/Packages/xml/ ; + #print "Opening $file for writing\n"; + open( OUTFILE, ">$file" ); -#now i will do the same thing but for the sarge build - #i now have all the files I need locally so I can do my opens properly. -open( MAIN, "/cluster/distro/debian/dists/sarge/main/binary-i386/Packages" ); -open( CONTRIB, "/cluster/distro/debian/dists/sarge/contrib/binary-i386/Packages"); -open( NONFREE, "/cluster/distro/debian/dists/sarge/non-free/binary-i386/Packages"); -open( NONMAIN, "/cluster/distro/debian-non-US/dists/sarge/non-US/main/binary-i386/Packages" ); -open( NONCONTRIB, "/cluster/distro/debian-non-US/dists/sarge/non-US/contrib/binary-i386/Packages"); -open( NONNONFREE, "/cluster/distro/debian-non-US/dists/sarge/non-US/non-free/binary-i386/Packages"); + #start by putting in the default stuff + print OUTFILE "\n"; + print OUTFILE "\n"; -open( LOCAL, "/cluster/debian/sarge/Packages"); -open( OUTFILE, ">/cluster/bcfg/images/debian-sarge/pkglist.xml" ); + #decrement the priority since we are going highest to lowest + $priority = $priority - 10; -print OUTFILE "\n"; -print OUTFILE "\n"; - -@mypackages = []; - -#get all the data from the local repo that we maintain. This is the highest precedence in package priority -while( $line = ){ + + #the loop that builds the actually file. + $known_package=0; + while( $line = ){ if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - push @mypackages, $basename; - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - print OUTFILE "\t\n" ; - $found =1; - } - } - } -} - - - -#now for all the regular packages in the mirrored repos - -$known_package=0; -while( $line =
){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line =
; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - push @mypackages, $basename; - } - $known_package = 0; - $found =1; - } - } + ($filler,$basename)=split( ' ', $line ); + + #Now to find the version of the package. + $found = 0; + while( !$found ){ + $line = ; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + if ( ! $known_package ){ + print OUTFILE "\t\n" ; + push @mypackages, $basename; } + $found =1; } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} + } + } + #end of file builder loop -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } + } + close( INFILE ); + print OUTFILE "\n\n"; + close( OUTFILE ); } -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} +#this is where I do clean up and set up for distributing the files to other +#servers. -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } +#clean up the temp files +foreach $file (@tmpfiles){ + #print "removing file: $file\n"; + system("rm -f $file"); } -print OUTFILE "\n\n"; +#get rid of old tarball +#print "Removing old tarball\n"; +system('rm -f /disks/debian/pkglists/bcfg2-packagelists.tgz'); +#create new tarball for distribution +#print "Creating new tarball\n"; +system('cd /disks/tmp/ ; tar czf /disks/debian/pkglists/bcfg2-packagelists.tgz bcfg2-packagelists 2&>1 >/dev/null' ); +#move the files into place on the local machine +#for testing purposes and also until we get netzero converted to 0.8 +#system('mv /disks/tmp/bcfg2-packagelists/*.xml /disks/tmp/bcfg2/Pkgmgr/'); +#for real +system('mv /disks/tmp/bcfg2-packagelists/*.xml /disks/bcfg2/Pkgmgr/'); -close(OUTFILE); -close(MAIN); -close(CONTRIB); -close(NONFREE); -close(NONMAIN); -close(NONCONTRIB); -close(NONNONFREE); -close(LOCAL); - +#final clean up +system('rmdir /disks/tmp/bcfg2-packagelists' ); -- cgit v1.2.3-1-g7c22