~hww3/caudium

ref: 93d5a75fba08db71b832716377276978cdd7888d caudium/tools/checkwebserver.pike -rwxr-xr-x 7.2 KiB
93d5a75fWilliam Welliver storage: method "None" was missing stop function. should fix error on shutdown 1 year, 1 month ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
#!/usr/bin/pike

// $Id$

/* This script is used to get running webserver version on a given set of sites
   Execute it with no arguments to get more informations */
/* given a list of websites in a file, will create a new file telling 
   which web server it runs */
      

// The timeout of the connection in seconds once we launch the connection  
// process to the remote server
#define TIMEOUT 10
// the MAX number of simultaneous connections we made
#define MAX_CONN 2000

// the number of connections at a given time
static int nb_connections = 0;
// the progress bar
static object bar;
// the number of sites we have checked so far
static int checked;
// the resuls are in this mapping
mapping(string:string) results = ([ ]);

void usage(string myname)
{
  string usage = sprintf("%s <inputfile> <outputfile>\n", myname);
  usage += "where inputfile is a file that contains a list of website\n";
  usage += "output will contains the same list with the webserver serving the site\n";
  usage += "if output already exists, it will be overwritten\n";
  write(usage);
}

inline void write2servers(string site, string servername)
{
  //write(sprintf("%s %s\n", site, servername));
  results += ([ site: servername ]);
  nb_connections--;
  checked++;
  bar->update(1);
}

class socket {

  // the server name we try to guess
  private string server = "";
  // the buffer for the data we read from the server
  private string buffer = "";
  // the site we are connected to
  private string site = "";
  private object fd;
  // the function to call on exit
  private object exit_cb;
  
  // note that a call_back function never (and can't) return anything 
  // by design
  // it is called by something external to this program (here the data
  // coming from the socket)
  private inline void read_callback(mixed id, string data)
  {
    // when we are called, we are not sure to get all the response from 
    // the server, we have to buffer it
    buffer += data;
  }
  
  // the server close the connection (as it should in HTTP non keepalived)
  private inline void close_callback(mixed id)
  {
    // first see if we get some data to work on
    if(sizeof(buffer) > 0)
    {
      int retpos = 0, oldretpos = 0;
      // here comes the headers parser
      // not very beautiful but should be fast
      while(retpos != -1)
      {
        retpos = search(buffer, "\r\n", oldretpos);
        string header_line = buffer[oldretpos..retpos-1];
        oldretpos = retpos+2;
	int pos = search(header_line, ":");
	if(pos != -1 && lower_case(header_line[..pos-1]) == "server")
	{
	  // remove one space in the server string
	  write2servers(site, String.trim_whites(header_line[pos+1..]));
	  remove_call_out(handle_timeout);
	  destruct(this_object());
	  return 0;
	}
      }
    }
    // fallback to unknow server
    write2servers(site, "null");
    // remove the timeout callout
    remove_call_out(handle_timeout);
    destruct(this_object());
  }
  
  inline void connect(int status, object _fd)
  {
    fd   = _fd;
    if(!status)
    {
     write2servers(site, "null"); 
     remove_call_out(handle_timeout);
     destruct(this_object());
     return 0;
    }
    fd->write(sprintf("HEAD / HTTP/1.1\r\nHost: %s\r\nConnection: Close\r\n\r\n", site));
    // first set socket to non blocking mode
    // this mean that the program (and so the thread) will not be 
    // blocked because we are waiting for something on the socket
    // read_callback() will be called automatically when data from
    // the server come to us, it is the same for the other callbacks
    fd->set_nonblocking(read_callback, 0, close_callback);
    // launch a new connection
    exit_cb->check_another_server();
  }

  inline void handle_timeout(object fd)
  {
    write2servers(site, "null");
    remove_call_out(handle_timeout);
    destruct(this_object());
  }

  inline void destroy()
  {
    catch(fd->close());
    if(find_call_out(exit_cb->check_another_server))
      remove_call_out(exit_cb->check_another_server);
    if(nb_connections < MAX_CONN)
    {
      exit_cb->check_another_server();
    }
  }

  inline void create(object cb, string _site)
  {
    exit_cb = cb;
    site = _site;
  }
  
};

class http {
 
  private array(string) lines = ({ });
  private object output_fd;
  private int nb_sites, start_time;
  private object async_dns_client;
  
  void create(array(string) _lines, object _output_fd, int _start_time)
  {
    output_fd = _output_fd;
    start_time = _start_time;
    checked = 0;
    //_lines = Array.uniq(_lines);
    foreach(_lines, string site)
    {
      if(site[0..6] == "http://")
        site = site[6..]; 
      if(sizeof(site) == 0)
        break;
      lines += ({ site });
    }
    nb_sites = sizeof(lines);
    bar = Tools.Install.ProgressBar("Checking", 0, nb_sites);
    async_dns_client = Protocols.DNS.async_client();
    check_another_server();
  }

  void end()
  {
    int elapsed_time = time() - start_time;
    write("\nTest finished\n");
    if(elapsed_time > 0)
      write("Scanned %d hosts in %d seconds: %f hosts/s\n", 
        checked, elapsed_time, (float)checked/elapsed_time);
    string output = "";
    foreach(indices(results), string indice)
      output += sprintf("%s %s\n", indice, results[indice]);
    output_fd->write(output);
    output_fd->close();
    exit(0);
  }

  inline void connect(string name, string|int ip)
  {
    if(ip)
    {
      // please note I write File and not FILE, there are two things different
      object fd = Stdio.File();
      object socket = socket(this_object(), name);
      int port = 80;
      // we connect asynchronously to the server, that is we don't wait for it 
      // to reply
      fd->async_connect(ip, port, socket->connect, fd);
      // a call_out allow you to make a delayed call to a function.
      // it is very useful for managing timeouts
      // test for the socket object because it may have be destroyed already
      if(socket)
        call_out(socket->handle_timeout, TIMEOUT, fd);
    }
    else
    {
      write2servers(name, "null");    
      if(nb_connections < MAX_CONN)
        check_another_server();
    }
  }
  
  inline void check_another_server()
  {
    //write("number of connections=%d\n", nb_connections);
    if(sizeof(lines) == 0)
    {
      if(nb_sites == checked)
        end();
    }
    else
    {
      string site = lines[0];
      // decrease the buffer
      lines = lines[1..];
      // resolve the name asynchronously
      async_dns_client->host_to_ip(site, connect);
      nb_connections++;
    }
  }
}

int main(int argc, array argv)
{
  if(argc != 3)
  {
    usage(argv[0]);
    return 1;
  }
  object infile = Stdio.File(argv[1], "r");
  // the number of sites we have to check
  int nb_sites;
  array(string) lines = ({ });
  foreach((infile->read() - "\r") / "\n", string line)
  {
    if(line && sizeof(line) > 0)
      lines += ({ line });
  }
  infile->close();
  if(!infile)
  {
    werror(sprintf("inputfile %s cannot be open for reading\n", argv[1]));
    return 2;
  }
  int j = 0;
  http(lines, Stdio.File(argv[2], "wc"), time());
  // if we return a negative value from main then we 
  // wait indefinitely (for the threads to finish their job)
  // don't put it something positive there or you won't let 
  // your threads the time to finish
  return -1;
}