/[safari]/get_book.sh
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /get_book.sh

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.3 by dpavlin, Mon Dec 15 00:26:57 2003 UTC revision 1.7 by dpavlin, Sun Feb 15 11:40:43 2004 UTC
# Line 1  Line 1 
1  #!/bin/sh  #!/bin/sh
2    
3  #export http_proxy=http://proxy:8080  # proxy settings (same as in firebird)
4    fping proxy && export http_proxy=http://proxy:8080
5    # user agent (same as in firebird)
6    ua="Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5) Gecko/20031225 Firebird/0.7"
7    
8    # wait between pages
9    export wait=120
10    
11    
12  if [ -z "$1" ] ; then  if [ -z "$1" ] ; then
13          echo "Usage: $0 ISBN"          echo "Usage: $0 ISBN"
14          exit 1          exit 1
15  fi  fi
16    
17  isbn=$1;  if [ -e orig ] ; then
18            echo "orig directory found. Resume download? [Y/n]"
19            read ans
20            if [ "$ans" = "n" ] ; then
21                    exit 1;
22            fi
23            mv orig/* .
24            rm -Rf orig
25    
26  wait=10          grep -l 'promo.asp' * | xargs -i rm {}
27            grep -l 'This is only a preview of the full book' * | xargs -i rm {}
28    fi
29    
30    isbn=$1;
31    
32  isbn2=`echo $isbn | sed 's/-//g'`  isbn2=`echo $isbn | sed 's/-//g'`
33    
34  function mirror() {  function mirror() {
35    
36            url="$1"
37    
38            file=`echo $url | sed -e s,http://[^?]*?,index.html?, -e s,#.*$,, -e s,/,%2F,g`
39            if [ -e "$file" ] ; then
40    #               echo "skip $url"
41                    return
42            fi
43    
44            if echo $url | grep '/index' >/dev/null ; then
45                    cookies=""
46                    echo -n "no login (index) "
47            elif echo $url | grep 'mode=toc' >/dev/null ; then
48                    cookies=""
49                    echo -n "no login (toc) "
50            else
51                    cookies="--load-cookies=/home/dpavlin/.phoenix/default/g6b45nt6.slt/cookies.txt"
52                    echo -n "with login "
53            fi
54            echo $url
55    
56          wget -p -nH -nc -k \          wget -p -nH -nc -k \
57                  --random-wait --wait=$wait -t 0 \                  -t 1 -U "$ua" \
58                  --load-cookies=/home/dpavlin/.phoenix/default/g6b45nt6.slt/cookies.txt \                  $cookies $url
                 -U "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5) Gecko/20031206 Firebird/0.7" \  
                 $1  
59    
60  #       -D safari.oreilly.com \  #       -D safari.oreilly.com \
61  #       -A 0-201-41975-0 \  #       -A 0-201-41975-0 \
62    
63            perl -e '$t=rand($ENV{wait} || 30);print "sleep for $t sec.\n"; sleep($t);'
64  }  }
65    
66  function geturl() {  function geturl() {
# Line 34  function geturl() { Line 73  function geturl() {
73                  -e 's/open=false/open=true/' | \                  -e 's/open=false/open=true/' | \
74                  grep '&s=1&b=1&f=1&t=1&c=1&u=1&r=&o=1' | \                  grep '&s=1&b=1&f=1&t=1&c=1&u=1&r=&o=1' | \
75                  grep $2 | \                  grep $2 | \
76                    grep -v "$2/[0-9][0-9][0-9][0-9][0-9][0-9][0-9]" | \
77                  sort -u >> in                  sort -u >> in
78  }  }
79    
# Line 43  function uniqurl() { Line 83  function uniqurl() {
83          grep 'view=[A-Z].*/index' in.tmp | sort -u >> in          grep 'view=[A-Z].*/index' in.tmp | sort -u >> in
84  }  }
85    
86  function checklogin() {  function mirror_in() {
87          if grep 'promo.asp' index.html* >/dev/null ; then          cat in | while read url ; do
88                  echo "WARNING: safari seems to logunt you as user. Aborting."                  mirror "$url"
89                  exit 1                  #sleep $wait
90          fi  
91                    if grep 'promo.asp' `ls -t index.html* | head -3` >/dev/null ; then
92                            echo "WARNING: safari seems to logout you as user. Aborting."
93                            exit 1
94                    fi
95    
96                    if grep -i '>Account locked<' `ls -t index.html* | head -3` >/dev/null ; then
97                            echo "WARNING: your safari account is locked. Aborting."
98                            exit 1
99                    fi
100    
101                    if grep -i 'session disabled' `ls -t index.html* | head -3` >/dev/null ; then
102                            echo "WARNING: your safari session is disabled. Aborting."
103                            exit 1
104                    fi
105    
106                    if grep -i 'This is only a preview of the full book' `ls -t index.html* | head -3` >/dev/null ; then
107                            echo "WARNING: you didn't add this book to your bookshelf!"
108                            exit 1
109                    fi
110            done
111  }  }
112    
113  echo > in  echo -n > in
114  mirror "http://safari.oreilly.com/?XmlId=$isbn"  mirror "http://safari.oreilly.com/?XmlId=$isbn"
115    
116  echo "extract URLs from first page..."  echo "extract URLs from first page..."
117  geturl "index.html?XmlId=$isbn" $isbn  geturl "index.html?XmlId=$isbn" $isbn
118  uniqurl  uniqurl
119    
120  mirror "-i in"  mirror_in
 checklogin  
121    
122  echo -n "extracting URLs [1]"  echo -n "extracting URLs [1]"
123  ls index.html* | while read file ; do  ls index.html* | while read file ; do
# Line 69  echo Line 128  echo
128    
129  uniqurl  uniqurl
130    
131  mirror "-i in"  mirror_in
 checklogin  
132    
133  echo > in  echo -n > in
134  echo -n "extracting URLs [2]"  echo -n "extracting URLs [2]"
135  ls index.html* | while read file ; do  ls index.html* | while read file ; do
136          echo -n "."          echo -n "."
# Line 81  done Line 139  done
139    
140  uniqurl  uniqurl
141    
142  mirror "-i in"  mirror_in
 checklogin  
143    
144  # convert links in html  # convert links in html
145  bn=`basename $0`  bn=`basename $0`

Legend:
Removed from v.1.3  
changed lines
  Added in v.1.7

  ViewVC Help
Powered by ViewVC 1.1.26