From fc9881381c8bb4d6e729532de2fe030b3ff418b3 Mon Sep 17 00:00:00 2001 From: Rick Bradshow Date: Mon, 27 Feb 2006 18:30:10 +0000 Subject: I updated all my repo scripts for the latest changes in the 0.8 version of the repo. these are still very specific to my repo, but should give a good idea of what is needed. git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1772 ce84e21b-d406-0410-9b95-82705330c041 --- tools/create-debian-pkglist.pl | 596 +++++++---------------------------------- tools/debian-build-cron.sh | 42 ++- tools/local-deb-repo-maker | 34 +-- 3 files changed, 130 insertions(+), 542 deletions(-) (limited to 'tools') diff --git a/tools/create-debian-pkglist.pl b/tools/create-debian-pkglist.pl index 8cf6409dc..1567f5aa2 100644 --- a/tools/create-debian-pkglist.pl +++ b/tools/create-debian-pkglist.pl @@ -1,536 +1,124 @@ #!/usr/bin/perl -#this is some setup that I have to do to make everything more localized below -#get all the packages/versions from security.debian.org. this is kinda hacky since we don't locally mirror that. I would like to for speed -#issues but since we are low on disk space I am not going to bother. -#these are the files that I will need to use to get the right info. +#some quicksetup to make sure everything is in place +system('mkdir -p /disks/tmp/bcfg2-packagelists' ); -system( "wget http://security.debian.org/dists/stable/updates/main/binary-i386/Packages -O /tmp/main.Packages -q" ); -system( "wget http://security.debian.org/dists/stable/updates/contrib/binary-i386/Packages -O /tmp/contrib.Packages -q" ); -system( "wget http://security.debian.org/dists/stable/updates/non-free/binary-i386/Packages -O /tmp/nonfree.Packages -q" ); -#i now have all the files I need locally so I can do my opens properly. +#pull the correct package lists from the security sites. +#this needs to be abstracted better +system( "wget http://security.debian.org/dists/stable/updates/main/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/security-main.Packages -q" ); +system( "wget http://security.debian.org/dists/stable/updates/contrib/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/security-contrib.Packages -q" ); +system( "wget http://security.debian.org/dists/stable/updates/non-free/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/security-nonfree.Packages -q" ); -open( MAIN, "/cluster/distro/debian/dists/stable/main/binary-i386/Packages" ); -open( CONTRIB, "/cluster/distro/debian/dists/stable/contrib/binary-i386/Packages"); -open( NONFREE, "/cluster/distro/debian/dists/stable/non-free/binary-i386/Packages"); -open( NONMAIN, "/cluster/distro/debian-non-US/dists/stable/non-US/main/binary-i386/Packages" ); -open( NONCONTRIB, "/cluster/distro/debian-non-US/dists/stable/non-US/contrib/binary-i386/Packages"); -open( NONNONFREE, "/cluster/distro/debian-non-US/dists/stable/non-US/non-free/binary-i386/Packages"); -open( SECMAIN, "/tmp/main.Packages"); -open( SECCONTRIB, "/tmp/contrib.Packages"); -open( SECNONFREE, "/tmp/nonfree.Packages"); +system('cat /disks/tmp/bcfg2-packagelists/security-main.Packages /disks/tmp/bcfg2-packagelists/security-contrib.Packages /disks/tmp/bcfg2-packagelists/security-nonfree.Packages > /disks/tmp/bcfg2-packagelists/debian-stable-security.Packages'); +system('rm /disks/tmp/bcfg2-packagelists/security-main.Packages /disks/tmp/bcfg2-packagelists/security-contrib.Packages /disks/tmp/bcfg2-packagelists/security-nonfree.Packages'); -open( LOCAL, "/cluster/debian/woody/Packages"); -open( OUTFILE, ">/cluster/bcfg/images/debian-3.0/pkglist.xml" ); +#pull the correct package lists from the security sites. +#this needs to be abstracted better +system( "wget http://volatile.debian.net/debian-volatile/dists/stable/volatile/main/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/volatile-main.Packages -q" ); +system( "wget http://volatile.debian.net/debian-volatile/dists/stable/volatile/contrib/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/volatile-contrib.Packages -q" ); +system( "wget http://volatile.debian.net/debian-volatile/dists/stable/volatile/non-free/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/volatile-nonfree.Packages -q" ); -print OUTFILE "\n"; -print OUTFILE "\n"; +system('cat /disks/tmp/bcfg2-packagelists/volatile-main.Packages /disks/tmp/bcfg2-packagelists/volatile-contrib.Packages /disks/tmp/bcfg2-packagelists/volatile-nonfree.Packages > /disks/tmp/bcfg2-packagelists/debian-stable-volatile.Packages'); +system('rm /disks/tmp/bcfg2-packagelists/volatile-main.Packages /disks/tmp/bcfg2-packagelists/volatile-contrib.Packages /disks/tmp/bcfg2-packagelists/volatile-nonfree.Packages'); -@mypackages = []; +#pull the correct package lists from the security sites. +#this needs to be abstracted better +system( "wget ftp://ftp.nerim.net/debian-marillat/dists/sarge/main/binary-i386/Packages -O /disks/tmp/bcfg2-packagelists/debian-sarge-mplayer.Packages -q" ); -#get all the data from the local repo that we maintain. This is the highest precedence in package priority -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - push @mypackages, $basename; - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - print OUTFILE "\t\n" ; - $found =1; - } - } - } -} - -#now i will do all the security packages and add them to the my packages if they aren't already there. If that makes sense. -#basically i am enforceing the priority by processing the package lists inorder. - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - #print $version."\n"; - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - push @mypackages, $basename; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - #print $version."\n"; - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - push @mypackages, $basename; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - #print $version."\n"; - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - push @mypackages, $basename; - } - $known_package = 0; - $found =1; - } - } - } -} - - -#now for all the regular packages in the mirrored repos - -$known_package=0; -while( $line =
){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line =
; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} +#this is to fix local files so that my naming hack thing will playout. +system('cp /disks/debian/sarge/Packages /disks/tmp/bcfg2-packagelists/debian-sarge-local.Packages'); -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} +system('cat /disks/distro/debian/dists/sarge/main/binary-i386/Packages /disks/distro/debian/dists/sarge/contrib/binary-i386/Packages /disks/distro/debian/dists/sarge/non-free/binary-i386/Packages > /disks/tmp/bcfg2-packagelists/debian-sarge-distro.Packages'); -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} +system('cat /disks/distro/debian-non-US/dists/sarge/non-US/main/binary-i386/Packages /disks/distro/debian-non-US/dists/sarge/non-US/contrib/binary-i386/Packages /disks/distro/debian-non-US/dists/sarge/non-US/non-free/binary-i386/Packages > /disks/tmp/bcfg2-packagelists/debian-sarge-nonUS.Packages'); -print OUTFILE "\n\n"; +#this is currently still a hack, because ordering is important. +#for future refernce you must do security and then local.. then the rest.. +@files = ( + "/disks/tmp/bcfg2-packagelists/debian-stable-volatile.Packages", + "/disks/tmp/bcfg2-packagelists/debian-stable-security.Packages", + "/disks/tmp/bcfg2-packagelists/debian-sarge-local.Packages", + "/disks/tmp/bcfg2-packagelists/debian-sarge-mplayer.Packages", + "/disks/tmp/bcfg2-packagelists/debian-sarge-distro.Packages", + "/disks/tmp/bcfg2-packagelists/debian-sarge-nonUS.Packages", +); +$priority = 89; +@tmpfiles = (); -close(OUTFILE); -close(MAIN); -close(CONTRIB); -close(NONFREE); -close(NONMAIN); -close(NONCONTRIB); -close(NONNONFREE); -close(SECMAIN); -close(SECCONTRIB); -close(SECNONFREE); -close(LOCAL); +#first come the security fixes +foreach $file ( @files ){ + push( @tmpfiles, $file ); + #first we open up the imput file + open( INFILE, "$file" ) or die("could not open $file\n"); + + #then we change the name and open the output file. + $file =~ s/Packages/xml/ ; + #print "Opening $file for writing\n"; + open( OUTFILE, ">$file" ); -#now i will do the same thing but for the sarge build - #i now have all the files I need locally so I can do my opens properly. -open( MAIN, "/cluster/distro/debian/dists/sarge/main/binary-i386/Packages" ); -open( CONTRIB, "/cluster/distro/debian/dists/sarge/contrib/binary-i386/Packages"); -open( NONFREE, "/cluster/distro/debian/dists/sarge/non-free/binary-i386/Packages"); -open( NONMAIN, "/cluster/distro/debian-non-US/dists/sarge/non-US/main/binary-i386/Packages" ); -open( NONCONTRIB, "/cluster/distro/debian-non-US/dists/sarge/non-US/contrib/binary-i386/Packages"); -open( NONNONFREE, "/cluster/distro/debian-non-US/dists/sarge/non-US/non-free/binary-i386/Packages"); + #start by putting in the default stuff + print OUTFILE "\n"; + print OUTFILE "\n"; -open( LOCAL, "/cluster/debian/sarge/Packages"); -open( OUTFILE, ">/cluster/bcfg/images/debian-sarge/pkglist.xml" ); + #decrement the priority since we are going highest to lowest + $priority = $priority - 10; -print OUTFILE "\n"; -print OUTFILE "\n"; - -@mypackages = []; - -#get all the data from the local repo that we maintain. This is the highest precedence in package priority -while( $line = ){ + + #the loop that builds the actually file. + $known_package=0; + while( $line = ){ if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - push @mypackages, $basename; - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - print OUTFILE "\t\n" ; - $found =1; - } - } - } -} - - - -#now for all the regular packages in the mirrored repos - -$known_package=0; -while( $line =
){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line =
; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - push @mypackages, $basename; - } - $known_package = 0; - $found =1; - } - } + ($filler,$basename)=split( ' ', $line ); + + #Now to find the version of the package. + $found = 0; + while( !$found ){ + $line = ; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + if ( ! $known_package ){ + print OUTFILE "\t\n" ; + push @mypackages, $basename; } + $found =1; } -} - -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} + } + } + #end of file builder loop -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } + } + close( INFILE ); + print OUTFILE "\n\n"; + close( OUTFILE ); } -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - #print "I already saw: $basename\n"; - $known_package =1; - } - } - $found = 0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } -} +#this is where I do clean up and set up for distributing the files to other +#servers. -$known_package=0; -while( $line = ){ - if( $line =~ /^Package:/ ){ - ($filler,$basename)=split( ' ', $line ); - #print $basename."\n"; - foreach $package ( @mypackages ){ - if( $basename eq $package ){ - $known_package =1; - #print "I already saw: $basename\n"; - } - } - $found=0; - while( !$found ){ - $line = ; - if( $line =~ /^Version:/ ){ - ($filler,$version)=split( ' ', $line ); - - if ( ! $known_package ){ - print OUTFILE "\t\n" ; - } - $known_package = 0; - $found =1; - } - } - } +#clean up the temp files +foreach $file (@tmpfiles){ + #print "removing file: $file\n"; + system("rm -f $file"); } -print OUTFILE "\n\n"; +#get rid of old tarball +#print "Removing old tarball\n"; +system('rm -f /disks/debian/pkglists/bcfg2-packagelists.tgz'); +#create new tarball for distribution +#print "Creating new tarball\n"; +system('cd /disks/tmp/ ; tar czf /disks/debian/pkglists/bcfg2-packagelists.tgz bcfg2-packagelists 2&>1 >/dev/null' ); +#move the files into place on the local machine +#for testing purposes and also until we get netzero converted to 0.8 +#system('mv /disks/tmp/bcfg2-packagelists/*.xml /disks/tmp/bcfg2/Pkgmgr/'); +#for real +system('mv /disks/tmp/bcfg2-packagelists/*.xml /disks/bcfg2/Pkgmgr/'); -close(OUTFILE); -close(MAIN); -close(CONTRIB); -close(NONFREE); -close(NONMAIN); -close(NONCONTRIB); -close(NONNONFREE); -close(LOCAL); - +#final clean up +system('rmdir /disks/tmp/bcfg2-packagelists' ); diff --git a/tools/debian-build-cron.sh b/tools/debian-build-cron.sh index 055f5b547..c03111412 100644 --- a/tools/debian-build-cron.sh +++ b/tools/debian-build-cron.sh @@ -1,15 +1,33 @@ #!/bin/sh -if /root/bin/rsync-debian-repo ; then - mv /cluster/bcfg/images/debian-3.0/pkglist.xml /cluster/bcfg/images/debian-3.0/pkglist.xml.old; - if /root/bin/create-debian-pkglist.pl ; then - /etc/init.d/bcfgd restart - else - mv /cluster/bcfg/images/debian-3.0/pkglist.xml.old /cluster/bcfg/images/debian-3.0/pkglist.xml; - echo "there was a problem with creating the new pkglist.xml"; - exit 1; - fi +# rsync all the repo stuff to keep it up to date +# this pulls the debian repo from mirror.mcs.anl.gov if the currect sync is finished. +# If it isn't finished it will just back off and try again when the script runs next time. + +if ! /root/bin/rsync-debian-repo ; then + echo "repo not synced "; +fi + +# now rebuild the local repo so that it will grab everything that was added today. +if ! /root/bin/local-deb-repo-maker ; then + echo "local repos are not up to date "; +fi + +#make a back up of the current pkglist so we can diff later +cp /disks/bcfg2/Pkgmgr/debian-sarge.xml /tmp/debian-sarge.xml.current + +if /root/bin/create-debian-pkglist.pl ; then + diff /tmp/debian-sarge.xml.current /disks/bcfg2/Pkgmgr/debian-sarge.xml + rm -f /tmp/debian-sarge.xml.current + + #this is mainly cause this dies from time to time.. so this is protection + /etc/init.d/apt-proxy restart else - echo "repo not synced and bcfg not updates"; - exit 1; + echo "there was a problem with creating the new pkglist.xml"; + exit 1; fi -exit 0; + +#this was problem-matic so I found that it was more realiable to just make its +#own cron job to restart every couple of days to reduce the memory consumption +#restart the server.. shouldn't be needed but memory leaks make it needed. +#/etc/init.d/bcfg2-server restart + diff --git a/tools/local-deb-repo-maker b/tools/local-deb-repo-maker index ab0771644..82bb9b0be 100644 --- a/tools/local-deb-repo-maker +++ b/tools/local-deb-repo-maker @@ -1,28 +1,10 @@ -#!/bin/sh +#!/bin/bash +# now build the Package files for everything in the sarge repo + +cd /disks/debian +dpkg-scanpackages sarge /dev/null > sarge/Packages +cd sarge +gzip -c Packages > Packages.gz + -HOSTNAME=`/bin/hostname` -#this is to pull the mplayer/avi stuff. -if [ ${HOSTNAME} == "debian-test" ]; then - cd /tmp - ncftpget ftp://ftp.nerim.net/debian-marillat/dists/stable/main/binary-i386/Packages - grep Package: Packages | awk '{print "ncftpget ftp://ftp.nerim.net/debian-marillat/dists/stable/main/binary-i386/" $2"\\*"}' > marillat-sync - cd /sandbox/woody/ - sh /tmp/marillat-sync - cd /sandbox - dpkg-scanpackages woody /dev/null > woody/Packages - cd woody - gzip -c Packages > Packages.gz -elif [ ${HOSTNAME} == "wonkarific.mcs.anl.gov" ]; then -#this is to pull the testing stuff. - - cd /tmp - ncftpget ftp://ftp.nerim.net/debian-marillat/dists/testing/main/binary-i386/Packages - grep Package: Packages | awk '{print "ncftpget ftp://ftp.nerim.net/debian-marillat/dists/testing/main/binary-i386/" $2"\\*"}' > marillat-sync - cd /sandbox/sarge/ - sh /tmp/marillat-sync - cd /sandbox - dpkg-scanpackages sarge /dev/null > sarge/Packages - cd sarge - gzip -c Packages > Packages.gz -fi -- cgit v1.2.3-1-g7c22