LWP FAQ

====================================================================

LWP FAQ by Paul Kulchenko (paulclinger@yahoo.com), updated 03/04/2000

====================================================================

How to get text file (http, ftp)?
How to get jpeg/gif/bmp file and return it?
How to access password protected file?
How to set up REFERER and other HTTP header parameters?
How to get specified part of file (first MAXSIZE bytes)?
How to get and set up cookies?
How to specify proxy servers?
How to check for redirect?
How to create parameters for POST method?
How to upload file?
How to access secure site (https://)
How to get information about remote file?
How to do "redirect" that works with both GET and POST? (Is it the same as "forward" does in Java?)

====================================================================
1. Как получить текстовый файл, используя http или ftp?


 use LWP::UserAgent;
 use CGI qw(header -no_debug);

 my $URL = 'http://www.yahoo.com/';
 my $res = LWP::UserAgent->new->request(new HTTP::Request GET => $URL);
 print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

2. Как скачать себе из сети файл jpeg/gif/bmp



 use LWP::UserAgent;
 use CGI qw(header -no_debug);
 $URL =
 'http://a100.g.akamaitech.net/7/100/70/0001/www.fool.com/art/new/butts/go99.
 gif';
 my $res = LWP::UserAgent->new->request(new HTTP::Request GET => $URL);
 binmode(STDOUT);
 print $res->is_success ? (header('image/gif'), $res->content)
                        : (header('text/html'), $res->status_line);

====================================================================

1.3. How to access password protected file?



 BEGIN {
   package RequestAgent;
   use LWP::UserAgent;
   @ISA = qw(LWP::UserAgent);

   sub new { LWP::UserAgent::new(@_); }
   sub get_basic_credentials { return 'user', 'password' }
 }
 use CGI qw(header -no_debug);

 my $res = RequestAgent->new->request(new HTTP::Request GET => $URL);
 print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.4. How to set up REFERER and other HTTP header parameters?



 use LWP::UserAgent;
 use HTTP::Headers;
 use CGI qw(header -no_debug);

 my $URL = 'http://localhost/cgi-bin/hello.cgi';
 my $res = LWP::UserAgent->new->request(
   new HTTP::Request(
     GET => $URL,
     new HTTP::Headers referer => 'http://www.yahoo.com'),
 );
 print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.5. How to get specified part of file (first MAXSIZE bytes)?



 use LWP::UserAgent;
 use CGI qw(header -no_debug);

 my $URL = 'http://www.yahoo.com/';
 my $MAXSIZE = 1024;

 print header;
 my $res = LWP::UserAgent->new->request(
   new HTTP::Request(GET => $URL), \&callback, $MAXSIZE);

 sub callback { my($data, $response, $protocol) = @_; print $data; die }

====================================================================

1.6. How to get and set up cookies?




 use LWP::UserAgent;
 use CGI qw(header -no_debug);
 use HTTP::Cookies;

 my $URL = 'http://mail.yahoo.com/';

 my $ua = new LWP::UserAgent;
 my $res = $ua->request(new HTTP::Request GET => $URL);
 my $cookie_jar = new HTTP::Cookies;
 $cookie_jar->extract_cookies($res);

 print header;
 if ($res->is_success) {
   my $req = new HTTP::Request GET => $URL;
   $cookie_jar->add_cookie_header($req);
   $res = $ua->request($req);
   print $res->is_success ? $res->as_string : $res->status_line;
 } else {
   print $res->status_line;
 }

====================================================================

1.7. How to specify proxy servers?



 use LWP::UserAgent;
 use CGI qw(header -no_debug);

 my $URL = 'http://www.yahoo.com/';
 my $ua = new LWP::UserAgent;

 $ua->proxy(['http', 'ftp'], 'http://proxy.sn.no:8001/');
 $ua->proxy('gopher', 'http://proxy.sn.no:8001/');

 my $res = $ua->request(new HTTP::Request GET => $URL);
 print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.8. How to check for redirect?



 use LWP::UserAgent;
 use CGI qw(header -no_debug);

 my $URL = 'http://www.yahoo.com/';
 my $res = LWP::UserAgent->new->request(new HTTP::Request GET => $URL);

 print header;
 print $res->request->url if $res->previous->is_redirect;

====================================================================

1.9. How to create parameters for POST method?



 use URI::URL;
 use HTTP::Request;
 use LWP::UserAgent;
 use CGI qw(header -no_debug);

 my $URL = 'http://yahoo.com/?login=mylogin&password=mypassword';
 my $uri = new URI $URL;
 my $method = 'POST';

 my $request;
 if (uc($method) eq 'POST') {
   my $query = $uri->query;
   (my $url = $uri->as_string) =~ s/\?$query$//;
   $request = new HTTP::Request ($method, $url);
   $request->header('Content-Type' => 'application/x-www-form-urlencoded');
   $request->content($query);
 } else {
   $request = new HTTP::Request ($method, $uri->as_string);
 };

 # add Host field as required in HTTP/1.1
 $request->header(Host => $uri->host_port) if $uri->scheme ne 'file';

 my $res = LWP::UserAgent->new->request($request);

 print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.10. How to upload file?



 use HTTP::Request::Common;
 use LWP::UserAgent;
 use CGI qw(header -no_debug);

 my $URL = 'http://localhost/cgi-bin/survey.cgi';
 my $req = POST $URL,
   Content_Type => 'form-data',
   Content      => 

     name       => 'Paul Kulchenko',
     email      => 'paulclinger@yahoo.com',
     surveyfile => ['./survey.dat'], # this file will be uploaded
   ];

 my $res = LWP::UserAgent->new->request($req);

 print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.11. How to access secure site (https://)



#!/usr/bin/perl -w
$test = "http://www.test.ru/account/secure.html";
use HTTP::Request::Common;
use LWP::UserAgent;
use CGI qw(header -no_debug);
$ua = LWP::UserAgent->new();

my $req = POST $test,
                [
        user => 'vovka',
    password => '123321',
                ];

$content = $ua->request($req)->as_string;
#print "content-type: text/html\n\n";
#print $content;
print $req->as_string;

====================================================================

1.12. How to get information about remote file?



 use LWP::UserAgent;
 use CGI qw(header -no_debug);

 my $url = 'http://yahoo.com/';
 my $res = LWP::UserAgent->new->request(HTTP::Request->new(HEAD => $url));

 print header;

 local $\ = "\n";
 if ($res->is_success) {
   $res->previous &&
   $res->previous->is_redirect and print 'redirected: ', $res->request->url;
   $res->last_modified         and print 'modified: ',
 scalar(localtime($res->last_modified));
   $res->content_length        and print 'size: ', $res->content_length;
 } else {
   print $res->status_line;
 }

====================================================================

1.13. How to do "redirect" that works with both GET and POST? (Is it the same as "forward" does in Java?)



 use LWP::UserAgent;
 use CGI qw(header -no_debug);

 my $URL = 'http://www.yahoo.com/';
 my $MAXSIZE = 1024;

 local $| = 'non buffered';

 print header;
 my $res = LWP::UserAgent->new->request(
   new HTTP::Request(GET => $URL),
   sub { my($data, $response, $protocol) = @_; print $data }, # <= callback
   $MAXSIZE,
 );

====================================================================