(Logical change 1.81)

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@391 ce84e21b-d406-0410-9b95-82705330c041

5 files changed, 586 insertions, 0 deletions

diff --git a/tools/create-debian-pkglist.pl b/tools/create-debian-pkglist.pl
index e69de29bb..8cf6409dc 100644
--- a/tools/create-debian-pkglist.pl
+++ b/tools/create-debian-pkglist.pl
@@ -0,0 +1,536 @@
+#!/usr/bin/perl
+
+#this is some setup that I have to do to make everything more localized below
+#get all the packages/versions from security.debian.org. this is kinda hacky since we don't locally mirror that. I would like to for speed
+#issues but since we are low on disk space I am not going to bother. 
+#these are the files that I will need to use to get the right info.
+
+system( "wget http://security.debian.org/dists/stable/updates/main/binary-i386/Packages -O /tmp/main.Packages -q" );
+system( "wget http://security.debian.org/dists/stable/updates/contrib/binary-i386/Packages -O /tmp/contrib.Packages -q" );
+system( "wget http://security.debian.org/dists/stable/updates/non-free/binary-i386/Packages -O /tmp/nonfree.Packages -q" );
+
+#i now have all the files I need locally so I can do my opens properly.
+
+open( MAIN, "/cluster/distro/debian/dists/stable/main/binary-i386/Packages" );
+open( CONTRIB, "/cluster/distro/debian/dists/stable/contrib/binary-i386/Packages");
+open( NONFREE, "/cluster/distro/debian/dists/stable/non-free/binary-i386/Packages");
+open( NONMAIN, "/cluster/distro/debian-non-US/dists/stable/non-US/main/binary-i386/Packages" );
+open( NONCONTRIB, "/cluster/distro/debian-non-US/dists/stable/non-US/contrib/binary-i386/Packages");
+open( NONNONFREE, "/cluster/distro/debian-non-US/dists/stable/non-US/non-free/binary-i386/Packages");
+open( SECMAIN, "/tmp/main.Packages");
+open( SECCONTRIB, "/tmp/contrib.Packages");
+open( SECNONFREE, "/tmp/nonfree.Packages");
+
+open( LOCAL, "/cluster/debian/woody/Packages");
+open( OUTFILE, ">/cluster/bcfg/images/debian-3.0/pkglist.xml" );
+
+print OUTFILE "<PackageList image='debian-stable'>\n";
+print OUTFILE "<Location uri='http://grandpoobah.mcs.anl.gov:8080/disks/cluster/disto/debian'>\n";
+
+@mypackages = [];
+
+#get all the data from the local repo that we maintain. This is the highest precedence in package priority                                                                                                                                                 
+while( $line = <LOCAL> ){
+    if( $line =~ /^Package:/ ){
+        ($filler,$basename)=split( ' ', $line );
+        push @mypackages, $basename;
+	$found=0;
+        while( !$found ){
+            $line = <LOCAL>;
+            if( $line =~ /^Version:/ ){
+                ($filler,$version)=split( ' ', $line );
+		
+                print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                $found =1;
+            }
+        }
+    }
+}
+
+#now i will do all the security packages and add them to the my packages if they aren't already there. If that makes sense. 
+#basically i am enforceing the priority by processing the package lists inorder.
+
+$known_package=0;
+while( $line = <SECMAIN> ){
+	if( $line =~ /^Package:/ ){
+		($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+			if( $basename eq $package ){	
+				#print "I already saw: $basename\n";
+				$known_package =1;
+			}
+		}
+		$found = 0;
+		while( !$found ){
+			$line = <SECMAIN>;
+			if( $line =~ /^Version:/ ){
+                		($filler,$version)=split( ' ', $line );
+				
+				#print $version."\n";
+				if ( ! $known_package ){
+					print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+					push @mypackages, $basename;
+				}
+				$known_package = 0;
+				$found =1;
+			}
+		}
+	}
+}
+
+$known_package=0;
+while( $line = <SECCONTRIB> ){
+	if( $line =~ /^Package:/ ){
+		($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+			if( $basename eq $package ){	
+				#print "I already saw: $basename\n";
+				$known_package =1;
+			}
+		}
+		$found = 0;
+		while( !$found ){
+			$line = <SECCONTRIB>;
+			if( $line =~ /^Version:/ ){
+                		($filler,$version)=split( ' ', $line );
+				
+				#print $version."\n";
+				if ( ! $known_package ){
+					print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+					push @mypackages, $basename;
+				}
+				$known_package = 0;
+				$found =1;
+			}
+		}
+	}
+}
+
+$known_package=0;
+while( $line = <SECNONFREE> ){
+	if( $line =~ /^Package:/ ){
+		($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+			if( $basename eq $package ){	
+				#print "I already saw: $basename\n";
+				$known_package =1;
+			}
+		}
+		$found = 0;
+		while( !$found ){
+			$line = <SECNONFREE>;
+			if( $line =~ /^Version:/ ){
+                		($filler,$version)=split( ' ', $line );
+				
+				#print $version."\n";
+				if ( ! $known_package ){
+					print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+					push @mypackages, $basename;
+				}
+				$known_package = 0;
+				$found =1;
+			}
+		}
+	}
+}
+
+
+#now for all the regular packages in the mirrored repos
+
+$known_package=0;
+while( $line = <MAIN> ){
+	if( $line =~ /^Package:/ ){
+		($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+			if( $basename eq $package ){	
+				#print "I already saw: $basename\n";
+				$known_package =1;
+			}
+		}
+		$found = 0;
+		while( !$found ){
+			$line = <MAIN>;
+			if( $line =~ /^Version:/ ){
+                		($filler,$version)=split( ' ', $line );
+				
+				if ( ! $known_package ){
+					print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+				}
+				$known_package = 0;
+				$found =1;
+			}
+		}
+	}
+}
+
+$known_package=0;
+while( $line = <CONTRIB> ){
+        if( $line =~ /^Package:/ ){
+                ($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+                        if( $basename eq $package ){
+                                $known_package =1;
+				#print "I already saw: $basename\n";
+                        }
+                }
+		$found=0;
+                while( !$found ){
+                        $line = <CONTRIB>;
+                        if( $line =~ /^Version:/ ){
+                                ($filler,$version)=split( ' ', $line );
+				
+				 if ( ! $known_package ){
+                                        print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                                }
+				$known_package = 0;
+                                $found =1;
+                        }
+                }
+        }
+}
+
+$known_package=0;
+while( $line = <NONFREE> ){
+        if( $line =~ /^Package:/ ){
+                ($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+                foreach $package ( @mypackages ){
+                        if( $basename eq $package ){
+                                $known_package =1;
+				#print "I already saw: $basename\n";
+                        }
+                }
+		$found=0;
+		while( !$found ){
+                        $line = <NONFREE>;
+                        if( $line =~ /^Version:/ ){
+                                ($filler,$version)=split( ' ', $line );
+				
+				if ( ! $known_package ){
+                                        print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                                }
+				$known_package = 0;
+                                $found =1;
+                        }
+                }
+        }
+}
+
+$known_package=0;
+while( $line = <NONMAIN> ){
+	if( $line =~ /^Package:/ ){
+		($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+			if( $basename eq $package ){	
+				#print "I already saw: $basename\n";
+				$known_package =1;
+			}
+		}
+		$found = 0;
+		while( !$found ){
+			$line = <NONMAIN>;
+			if( $line =~ /^Version:/ ){
+                		($filler,$version)=split( ' ', $line );
+				
+				if ( ! $known_package ){
+					print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+				}
+				$known_package = 0;
+				$found =1;
+			}
+		}
+	}
+}
+
+$known_package=0;
+while( $line = <NONCONTRIB> ){
+        if( $line =~ /^Package:/ ){
+                ($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+                        if( $basename eq $package ){
+                                $known_package =1;
+				#print "I already saw: $basename\n";
+                        }
+                }
+		$found=0;
+                while( !$found ){
+                        $line = <NONCONTRIB>;
+                        if( $line =~ /^Version:/ ){
+                                ($filler,$version)=split( ' ', $line );
+				
+				 if ( ! $known_package ){
+                                        print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                                }
+				$known_package = 0;
+                                $found =1;
+                        }
+                }
+        }
+}
+
+$known_package=0;
+while( $line = <NONNONFREE> ){
+        if( $line =~ /^Package:/ ){
+                ($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+                foreach $package ( @mypackages ){
+                        if( $basename eq $package ){
+                                $known_package =1;
+				#print "I already saw: $basename\n";
+                        }
+                }
+		$found=0;
+		while( !$found ){
+                        $line = <NONNONFREE>;
+                        if( $line =~ /^Version:/ ){
+                                ($filler,$version)=split( ' ', $line );
+				
+				if ( ! $known_package ){
+                                        print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                                }
+				$known_package = 0;
+                                $found =1;
+                        }
+                }
+        }
+}
+
+print OUTFILE "</Location>\n</PackageList>\n";
+
+
+
+close(OUTFILE);
+close(MAIN);
+close(CONTRIB);
+close(NONFREE);
+close(NONMAIN);
+close(NONCONTRIB);
+close(NONNONFREE);
+close(SECMAIN);
+close(SECCONTRIB);
+close(SECNONFREE);
+close(LOCAL);
+
+
+#now i will do the same thing but for the sarge build
+ #i now have all the files I need locally so I can do my opens properly.
+
+open( MAIN, "/cluster/distro/debian/dists/sarge/main/binary-i386/Packages" );
+open( CONTRIB, "/cluster/distro/debian/dists/sarge/contrib/binary-i386/Packages");
+open( NONFREE, "/cluster/distro/debian/dists/sarge/non-free/binary-i386/Packages");
+open( NONMAIN, "/cluster/distro/debian-non-US/dists/sarge/non-US/main/binary-i386/Packages" );
+open( NONCONTRIB, "/cluster/distro/debian-non-US/dists/sarge/non-US/contrib/binary-i386/Packages");
+open( NONNONFREE, "/cluster/distro/debian-non-US/dists/sarge/non-US/non-free/binary-i386/Packages");
+
+open( LOCAL, "/cluster/debian/sarge/Packages");
+open( OUTFILE, ">/cluster/bcfg/images/debian-sarge/pkglist.xml" );
+
+print OUTFILE "<PackageList image='debian-sarge'>\n";
+print OUTFILE "<Location uri='http://grandpoobah.mcs.anl.gov:8080/disks/cluster/disto/debian'>\n";
+
+@mypackages = [];
+
+#get all the data from the local repo that we maintain. This is the highest precedence in package priority                                                                                                                                                 
+while( $line = <LOCAL> ){
+    if( $line =~ /^Package:/ ){
+        ($filler,$basename)=split( ' ', $line );
+        push @mypackages, $basename;
+	$found=0;
+        while( !$found ){
+            $line = <LOCAL>;
+            if( $line =~ /^Version:/ ){
+                ($filler,$version)=split( ' ', $line );
+                print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                $found =1;
+            }
+        }
+    }
+}
+
+
+
+#now for all the regular packages in the mirrored repos
+
+$known_package=0;
+while( $line = <MAIN> ){
+	if( $line =~ /^Package:/ ){
+		($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+			if( $basename eq $package ){	
+				#print "I already saw: $basename\n";
+				$known_package =1;
+			}
+		}
+		$found = 0;
+		while( !$found ){
+			$line = <MAIN>;
+			if( $line =~ /^Version:/ ){
+                		($filler,$version)=split( ' ', $line );
+				
+				if ( ! $known_package ){
+					print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+					push @mypackages, $basename;
+				}
+				$known_package = 0;
+				$found =1;
+			}
+		}
+	}
+}
+
+$known_package=0;
+while( $line = <CONTRIB> ){
+        if( $line =~ /^Package:/ ){
+                ($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+                        if( $basename eq $package ){
+                                $known_package =1;
+				#print "I already saw: $basename\n";
+                        }
+                }
+		$found=0;
+                while( !$found ){
+                        $line = <CONTRIB>;
+                        if( $line =~ /^Version:/ ){
+                                ($filler,$version)=split( ' ', $line );
+				
+				 if ( ! $known_package ){
+                                        print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                                }
+				$known_package = 0;
+                                $found =1;
+                        }
+                }
+        }
+}
+
+$known_package=0;
+while( $line = <NONFREE> ){
+        if( $line =~ /^Package:/ ){
+                ($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+                foreach $package ( @mypackages ){
+                        if( $basename eq $package ){
+                                $known_package =1;
+				#print "I already saw: $basename\n";
+                        }
+                }
+		$found=0;
+		while( !$found ){
+                        $line = <NONFREE>;
+                        if( $line =~ /^Version:/ ){
+                                ($filler,$version)=split( ' ', $line );
+				
+				if ( ! $known_package ){
+                                        print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                                }
+				$known_package = 0;
+                                $found =1;
+                        }
+                }
+        }
+}
+
+$known_package=0;
+while( $line = <NONMAIN> ){
+	if( $line =~ /^Package:/ ){
+		($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+			if( $basename eq $package ){	
+				#print "I already saw: $basename\n";
+				$known_package =1;
+			}
+		}
+		$found = 0;
+		while( !$found ){
+			$line = <NONMAIN>;
+			if( $line =~ /^Version:/ ){
+                		($filler,$version)=split( ' ', $line );
+				
+				if ( ! $known_package ){
+					print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+				}
+				$known_package = 0;
+				$found =1;
+			}
+		}
+	}
+}
+
+$known_package=0;
+while( $line = <NONCONTRIB> ){
+        if( $line =~ /^Package:/ ){
+                ($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+		foreach $package ( @mypackages ){
+                        if( $basename eq $package ){
+                                $known_package =1;
+				#print "I already saw: $basename\n";
+                        }
+                }
+		$found=0;
+                while( !$found ){
+                        $line = <NONCONTRIB>;
+                        if( $line =~ /^Version:/ ){
+                                ($filler,$version)=split( ' ', $line );
+				
+				 if ( ! $known_package ){
+                                        print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                                }
+				$known_package = 0;
+                                $found =1;
+                        }
+                }
+        }
+}
+
+$known_package=0;
+while( $line = <NONNONFREE> ){
+        if( $line =~ /^Package:/ ){
+                ($filler,$basename)=split( ' ', $line );
+		#print $basename."\n";
+                foreach $package ( @mypackages ){
+                        if( $basename eq $package ){
+                                $known_package =1;
+				#print "I already saw: $basename\n";
+                        }
+                }
+		$found=0;
+		while( !$found ){
+                        $line = <NONNONFREE>;
+                        if( $line =~ /^Version:/ ){
+                                ($filler,$version)=split( ' ', $line );
+				
+				if ( ! $known_package ){
+                                        print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ;
+                                }
+				$known_package = 0;
+                                $found =1;
+                        }
+                }
+        }
+}
+
+print OUTFILE "</Location>\n</PackageList>\n";
+
+
+
+close(OUTFILE);
+close(MAIN);
+close(CONTRIB);
+close(NONFREE);
+close(NONMAIN);
+close(NONCONTRIB);
+close(NONNONFREE);
+close(LOCAL);
+	
diff --git a/tools/debian-build-cron.sh b/tools/debian-build-cron.sh
index e69de29bb..055f5b547 100644
--- a/tools/debian-build-cron.sh
+++ b/tools/debian-build-cron.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+if /root/bin/rsync-debian-repo ; then
+	mv /cluster/bcfg/images/debian-3.0/pkglist.xml /cluster/bcfg/images/debian-3.0/pkglist.xml.old;
+	if /root/bin/create-debian-pkglist.pl ; then
+		/etc/init.d/bcfgd restart
+	else
+		mv /cluster/bcfg/images/debian-3.0/pkglist.xml.old /cluster/bcfg/images/debian-3.0/pkglist.xml;
+		echo "there was a problem with creating the new pkglist.xml";
+		exit 1;
+	fi
+else
+	echo "repo not synced and bcfg not updates";
+	exit 1;
+fi
+exit 0;
diff --git a/tools/debian-repo-cleaner b/tools/debian-repo-cleaner
index e69de29bb..3962f6f89 100644
--- a/tools/debian-repo-cleaner
+++ b/tools/debian-repo-cleaner
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+for i in alpha arm hppa ia64 m68k mipsel powerpc s390 sparc mips; do find /cluster/distro/debian -name \*${i}.deb | xargs rm -f; done
diff --git a/tools/local-deb-repo-maker b/tools/local-deb-repo-maker
index e69de29bb..ab0771644 100644
--- a/tools/local-deb-repo-maker
+++ b/tools/local-deb-repo-maker
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+HOSTNAME=`/bin/hostname`
+
+#this is to pull the mplayer/avi stuff.
+if [ ${HOSTNAME} == "debian-test" ]; then
+    cd /tmp
+    ncftpget ftp://ftp.nerim.net/debian-marillat/dists/stable/main/binary-i386/Packages
+    grep Package: Packages | awk '{print "ncftpget ftp://ftp.nerim.net/debian-marillat/dists/stable/main/binary-i386/" $2"\\*"}' > marillat-sync
+    cd /sandbox/woody/
+    sh /tmp/marillat-sync
+    cd /sandbox
+    dpkg-scanpackages woody /dev/null > woody/Packages
+    cd woody
+    gzip -c Packages > Packages.gz
+elif [ ${HOSTNAME} == "wonkarific.mcs.anl.gov" ]; then
+#this is to pull the testing stuff.
+    
+    cd /tmp
+    ncftpget ftp://ftp.nerim.net/debian-marillat/dists/testing/main/binary-i386/Packages
+    grep Package: Packages | awk '{print "ncftpget ftp://ftp.nerim.net/debian-marillat/dists/testing/main/binary-i386/" $2"\\*"}' > marillat-sync
+    cd /sandbox/sarge/
+    sh /tmp/marillat-sync
+    cd /sandbox
+    dpkg-scanpackages sarge /dev/null > sarge/Packages
+    cd sarge
+    gzip -c Packages > Packages.gz
+fi
diff --git a/tools/rsync-debian-repo b/tools/rsync-debian-repo
index e69de29bb..c7902b3ef 100644
--- a/tools/rsync-debian-repo
+++ b/tools/rsync-debian-repo
@@ -0,0 +1,4 @@
+#!/bin/sh
+/usr/bin/rsync -a --exclude="**/*alpha*" --exclude="**/*arm*" --exclude="**/*hppa*" --exclude="**/*ia64*" --exclude="**/*m68k*" --exclude="**/*mipsel*" --exclude="**/*powerpc*" --exclude="**/*s390*" --exclude="**/*sparc*" --exclude="**/*mips*" rsync://mirror.mcs.anl.gov/debian/ /cluster/distro/debian/
+/usr/bin/rsync -a --exclude="**/*alpha*" --exclude="**/*arm*" --exclude="**/*hppa*" --exclude="**/*ia64*" --exclude="**/*m68k*" --exclude="**/*mipsel*" --exclude="**/*powerpc*" --exclude="**/*s390*" --exclude="**/*sparc*" --exclude="**/*mips*" rsync://non-us.debian.org/debian-non-US/ /cluster/distro/debian-non-US/
+rsync -a rsync://debian-test/deb-repo/ /cluster/debian/woody/