Chapter 16 - Networking

WebClient Download Page

WebClient wc = new WebClient { Proxy = null };
wc.DownloadFile ("http://www.albahari.com/nutshell/code.aspx", "code.htm");

OpenHtml ("code.htm");

void OpenHtml (string location)
{
  if (RuntimeInformation.IsOSPlatform (OSPlatform.Windows))
    Process.Start (new ProcessStartInfo ("cmd", $"/c start {location}"));
  else if (RuntimeInformation.IsOSPlatform (OSPlatform.Linux))
    Process.Start ("xdg-open", location); // Desktop Linux
  else throw new Exception ("Platform-specific code needed to open URL.");
}

WebRequest and WebResponse

void DownloadPage()
{
  WebRequest req = WebRequest.Create
                  ("http://www.albahari.com/nutshell/code.html");
  req.Proxy = null;
  using (WebResponse res = req.GetResponse())
  using (Stream rs = res.GetResponseStream())
  using (FileStream fs = File.Create ("code_sync.html"))
    rs.CopyTo (fs);
}

async Task DownloadPageAsync()
{
  WebRequest req = WebRequest.Create
                  ("http://www.albahari.com/nutshell/code.html");
  req.Proxy = null;
  using (WebResponse res = await req.GetResponseAsync())
  using (Stream rs = res.GetResponseStream())
  using (FileStream fs = File.Create ("code_async.html"))
    await rs.CopyToAsync (fs);
}

DownloadPage();
await DownloadPageAsync();

foreach (var file in Directory.EnumerateFiles (".", "*.html"))
  Console.WriteLine ($"{file} {new FileInfo (file).Length} bytes");

HttpClient - simple request

string html = await new HttpClient().GetStringAsync ("http://linqpad.net");
html.Dump();

HttpClient - parallel downloads

var client = new HttpClient();
var task1 = client.GetStringAsync ("http://www.linqpad.net");
var task2 = client.GetStringAsync ("http://www.albahari.com");

(await task1).Length.Dump ("First page length");
(await task2).Length.Dump ("Second page length");

HttpClient - response messages

var client = new HttpClient();
// The GetAsync method also accepts a CancellationToken.
HttpResponseMessage response = await client.GetAsync ("http://www.linqpad.net");
response.EnsureSuccessStatusCode();
string html = await response.Content.ReadAsStringAsync();

HttpClient - uploading data

var client = new HttpClient (new HttpClientHandler { UseProxy = false });
var request = new HttpRequestMessage (
  HttpMethod.Post, "http://www.albahari.com/EchoPost.aspx");
request.Content = new StringContent ("This is a test");
HttpResponseMessage response = await client.SendAsync (request);
response.EnsureSuccessStatusCode();
Console.WriteLine (await response.Content.ReadAsStringAsync());

HttpClient - Using HttpMessageHandler for mocking

async Task Main()
{
  var mocker = new MockHandler (request =>
    new HttpResponseMessage (HttpStatusCode.OK)
    {
      Content = new StringContent ("You asked for " + request.RequestUri)
    });

  var client = new HttpClient (mocker);
  var response = await client.GetAsync ("http://www.linqpad.net");
  string result = await response.Content.ReadAsStringAsync();
  
  Assert.AreEqual ("You asked for http://www.linqpad.net/", result);
}

class MockHandler : HttpMessageHandler
{
  Func<HttpRequestMessage, HttpResponseMessage> _responseGenerator;

  public MockHandler
    (Func<HttpRequestMessage, HttpResponseMessage> responseGenerator)
  {
    _responseGenerator = responseGenerator;
  }

  protected override Task<HttpResponseMessage> SendAsync
    (HttpRequestMessage request, CancellationToken cancellationToken)
  {
    cancellationToken.ThrowIfCancellationRequested();
    var response = _responseGenerator (request);
    response.RequestMessage = request;
    return Task.FromResult (response);
  }
}

static class Assert
{
  public static void AreEqual (object o1, object o2)
  {
    if (!Equals (o1, o2)) throw new Exception ("Objects are not equal");    
  }    
}

HttpClient - Chaining handlers with DelegatingHandler

async Task Main()
{
  var mocker = new MockHandler (request =>
    new HttpResponseMessage (HttpStatusCode.OK)
    {
      Content = new StringContent ("You asked for " + request.RequestUri)
    });

  var logger = new LoggingHandler (mocker);

  var client = new HttpClient (logger);
  var response = await client.GetAsync ("http://www.linqpad.net");
  string result = await response.Content.ReadAsStringAsync();
  
  Assert.AreEqual ("You asked for http://www.linqpad.net/", result);
}

class MockHandler : HttpMessageHandler
{
  Func<HttpRequestMessage, HttpResponseMessage> _responseGenerator;

  public MockHandler
    (Func<HttpRequestMessage, HttpResponseMessage> responseGenerator)
  {
    _responseGenerator = responseGenerator;
  }

  protected override Task<HttpResponseMessage> SendAsync
    (HttpRequestMessage request, CancellationToken cancellationToken)
  {
    cancellationToken.ThrowIfCancellationRequested();
    var response = _responseGenerator (request);
    response.RequestMessage = request;
    return Task.FromResult (response);
  }
}

class LoggingHandler : DelegatingHandler
{
  public LoggingHandler (HttpMessageHandler nextHandler)
  {
    InnerHandler = nextHandler;
  }

  protected async override Task<HttpResponseMessage> SendAsync
    (HttpRequestMessage request, CancellationToken cancellationToken)
  {
    Console.WriteLine ("Requesting: " + request.RequestUri);
    var response = await base.SendAsync (request, cancellationToken);
    Console.WriteLine ("Got response: " + response.StatusCode);
    return response;
  }
}

static class Assert
{
  public static void AreEqual (object o1, object o2)
  {
    if (!Equals (o1, o2)) throw new Exception ("Objects are not equal");    
  }    
}

EXTRA - HttpClient With Progress

async Task Main()
{
  var linqPadProgressBar = new Util.ProgressBar ("Download progress").Dump();
  
  var progress = new Progress<double>();
  
  progress.ProgressChanged += (sender, value) =>
    linqPadProgressBar.Percent = (int) value;

  var cancellationToken = new CancellationTokenSource();

  using var destination = File.OpenWrite ("LINQPad6Setup.exe");
  await DownloadFileAsync ("http://www.linqpad.net/GetFile.aspx?LINQPad6Setup.exe", destination, progress, default);
}

// Based on: http://stackoverflow.com/q/21169573/141172
//           http://stackoverflow.com/q/230128/141172

HttpClient client = new HttpClient();

async Task CopyStreamWithProgressAsync (Stream input, Stream output, long total, IProgress<double> progress, CancellationToken token)
{
  const int IO_BUFFER_SIZE = 8 * 1024; // Optimal size depends on your scenario

  // Expected size of input stream may be known from an HTTP header when reading from HTTP. Other streams may have their
  // own protocol for pre-reporting expected size.

  var canReportProgress = total != -1 && progress != null;
  var totalRead = 0L;
  byte[] buffer = new byte [IO_BUFFER_SIZE];
  int read;
  
  while ((read = await input.ReadAsync (buffer, 0, buffer.Length)) > 0)
  {
    token.ThrowIfCancellationRequested();
    await output.WriteAsync (buffer, 0, read);
    totalRead += read;
    if (canReportProgress)
      progress.Report ((totalRead * 1d) / (total * 1d) * 100);
  }
}
 
 async Task DownloadFileAsync (string url, Stream destination, IProgress<double> progress, CancellationToken token)
{
  var response = await client.GetAsync (url, HttpCompletionOption.ResponseHeadersRead, token);

  if (!response.IsSuccessStatusCode)
    throw new Exception (string.Format ("The request returned with HTTP status code {0}", response.StatusCode));

  var total = response.Content.Headers.ContentLength.HasValue ? response.Content.Headers.ContentLength.Value : -1L;

  using var source = await response.Content.ReadAsStreamAsync();
  
  await CopyStreamWithProgressAsync(source, destination, total, progress, token);
}

Exception handling

WebClient wc = new WebClient { Proxy = null };
try
{
  string s = wc.DownloadString ("http://www.albahari.com/notthere");
}
catch (WebException ex)
{
  if (ex.Status == WebExceptionStatus.NameResolutionFailure)
    Console.WriteLine ("Bad domain name");
  else if (ex.Status == WebExceptionStatus.ProtocolError)
  {
    HttpWebResponse response = (HttpWebResponse)ex.Response;
    Console.WriteLine (response.StatusDescription);      // "Not Found"
    if (response.StatusCode == HttpStatusCode.NotFound)
      Console.WriteLine ("Not there!");                  // "Not there!"
  }
  else throw;
}

Simple Address Parsing

IPAddress a1 = new IPAddress (new byte[] { 101, 102, 103, 104 });
IPAddress a2 = IPAddress.Parse ("101.102.103.104");
Console.WriteLine (a1.Equals (a2));                     // True
Console.WriteLine (a1.AddressFamily);                   // InterNetwork

IPAddress a3 = IPAddress.Parse
  ("[3EA0:FFFF:198A:E4A3:4FF2:54fA:41BC:8D31]");
Console.WriteLine (a3.AddressFamily);   // InterNetworkV6

Console.WriteLine();
Console.WriteLine("Address with port:");
IPAddress a = IPAddress.Parse ("101.102.103.104");
IPEndPoint ep = new IPEndPoint (a, 222);           // Port 222
Console.WriteLine (ep.ToString());                 // 101.102.103.104:222

Uri Usage

Uri info = new Uri ("http://www.domain.com:80/info/");
Uri page = new Uri ("http://www.domain.com/info/page.html");

Console.WriteLine (info.Host);     // www.domain.com
Console.WriteLine (info.Port);     // 80
Console.WriteLine (page.Port);     // 80  (Uri knows the default HTTP port)

Console.WriteLine (info.IsBaseOf (page));         // True
Uri relative = info.MakeRelativeUri (page);
Console.WriteLine (relative.IsAbsoluteUri);       // False
Console.WriteLine (relative.ToString());          // page.html

HTTP Headers

WebClient wc = new WebClient { Proxy = null };
wc.Headers.Add ("CustomHeader", "JustPlaying/1.0");
wc.DownloadString ("http://www.oreilly.com");

foreach (string name in wc.ResponseHeaders.Keys)
  Console.WriteLine (name + "=" + wc.ResponseHeaders [name]);

Query strings

WebClient wc = new WebClient { Proxy = null };
wc.QueryString.Add ("q", "WebClient");     // Search for "WebClient"
wc.QueryString.Add ("hl", "fr");           // Display page in French
wc.DownloadFile ("http://www.google.com/search", "results.html");
OpenHtml ("results.html");

void OpenHtml (string location)
{
  if (RuntimeInformation.IsOSPlatform (OSPlatform.Windows))
    Process.Start (new ProcessStartInfo ("cmd", $"/c start {location}"));
  else if (RuntimeInformation.IsOSPlatform (OSPlatform.Linux))
    Process.Start ("xdg-open", location); // Desktop Linux
  else throw new Exception ("Platform-specific code needed to open URL.");
}

EscapeDataString

string search = Uri.EscapeDataString ("(WebClient OR HttpClient)");
string language = Uri.EscapeDataString ("fr");
string requestURI = "http://www.google.com/search?q=" + search +
                    "&hl=" + language;

new WebClient { Proxy = null }.DownloadFile (requestURI, "results.html");
OpenHtml ("results.html");

void OpenHtml (string location)
{
  if (RuntimeInformation.IsOSPlatform (OSPlatform.Windows))
    Process.Start (new ProcessStartInfo ("cmd", $"/c start {location}"));
  else if (RuntimeInformation.IsOSPlatform (OSPlatform.Linux))
    Process.Start ("xdg-open", location); // Desktop Linux
  else throw new Exception ("Platform-specific code needed to open URL.");
}

Uploading form data - with WebClient

WebClient wc = new WebClient { Proxy = null };

var data = new System.Collections.Specialized.NameValueCollection();
data.Add ("Name", "Joe Albahari");
data.Add ("Company", "O'Reilly");

byte[] result = wc.UploadValues ("http://www.albahari.com/EchoPost.aspx",
                                 "POST", data);

Console.WriteLine (Encoding.UTF8.GetString (result));

Uploading form data - with WebRequest

var req = WebRequest.Create ("http://www.albahari.com/EchoPost.aspx");
req.Proxy = null;
req.Method = "POST";
req.ContentType = "application/x-www-form-urlencoded";

string reqString = "Name=Joe+Albahari&Company=O'Reilly";
byte[] reqData = Encoding.UTF8.GetBytes (reqString);
req.ContentLength = reqData.Length;

using (Stream reqStream = req.GetRequestStream())
  reqStream.Write (reqData, 0, reqData.Length);

using (WebResponse res = req.GetResponse())
using (Stream resSteam = res.GetResponseStream())
using (StreamReader sr = new StreamReader (resSteam))
  Console.WriteLine (sr.ReadToEnd());

Uploading form data - with HttpClient

string uri = "http://www.albahari.com/EchoPost.aspx";
var client = new HttpClient();
var dict = new Dictionary<string,string> 
{
    { "Name", "Joe Albahari" },
    { "Company", "O'Reilly" }
};
var values = new FormUrlEncodedContent (dict);
var response = await client.PostAsync (uri, values);
response.EnsureSuccessStatusCode();
Console.WriteLine (await response.Content.ReadAsStringAsync());

Cookies

var cc = new CookieContainer();

var request = (HttpWebRequest)WebRequest.Create ("http://www.google.com");
request.Proxy = null;
request.CookieContainer = cc;
using (var response = (HttpWebResponse)request.GetResponse())
{
  foreach (Cookie c in response.Cookies)
  {
    Console.WriteLine (" Name:   " + c.Name);
    Console.WriteLine (" Value:  " + c.Value);
    Console.WriteLine (" Path:   " + c.Path);
    Console.WriteLine (" Domain: " + c.Domain);
  }
  // Read response stream...
}

One Response HTTP Server

static void Main()
{
  using var server = new SimpleHttpServer();

  WebClient wc = new WebClient();          // Make a client request.

  Console.WriteLine (wc.DownloadString
    ("http://localhost:51111/MyApp/Request.txt"));
}

class SimpleHttpServer : IDisposable
{
  readonly HttpListener listener = new HttpListener();
  
  public SimpleHttpServer() => ListenAsync();  
  async void ListenAsync()
  {
    listener.Prefixes.Add ("http://localhost:51111/MyApp/");  // Listen on
    listener.Start();                                         // port 51111.

    // Await a client request:
    HttpListenerContext context = await listener.GetContextAsync();

    // Respond to the request:
    string msg = "You asked for: " + context.Request.RawUrl;
    context.Response.ContentLength64 = Encoding.UTF8.GetByteCount (msg);
    context.Response.StatusCode = (int)HttpStatusCode.OK;

    using (Stream s = context.Response.OutputStream)
    using (StreamWriter writer = new StreamWriter (s))
      await writer.WriteAsync (msg);
  }

  public void Dispose() => listener.Close();
}

Simple HTTP Server

static void Main()
{
  // Listen on port 51111, serving files in d:\webroot:
  var server = new WebServer ("http://localhost:51111/", Path.Combine(TempDirectory, "webroot"));
  try
  {
    server.Start();
    // If running in LINQPad, stop the query manually:
    Console.WriteLine ("Server running... press Enter to stop");
    Console.ReadLine();
  }
  finally { server.Stop(); }
}


class WebServer
{
  HttpListener _listener;
  string _baseFolder;      // Your web page folder.

  public WebServer (string uriPrefix, string baseFolder)
  {
    _listener = new HttpListener();
    _listener.Prefixes.Add (uriPrefix);
    _baseFolder = baseFolder;
  }

  public async void Start()
  {
    _listener.Start();
    while (true)
      try
      {
        var context = await _listener.GetContextAsync();
        Task.Run (() => ProcessRequestAsync (context));
      }
      catch (HttpListenerException) { break; }   // Listener stopped.
      catch (InvalidOperationException) { break; }   // Listener stopped.
  }

  public void Stop() { _listener.Stop(); }

  async void ProcessRequestAsync (HttpListenerContext context)
  {
    try
    {
      string filename = Path.GetFileName (context.Request.RawUrl);
      string path = Path.Combine (_baseFolder, filename);
      byte[] msg;
      if (!File.Exists (path))
      {
        Console.WriteLine ("Resource not found: " + path);
        context.Response.StatusCode = (int)HttpStatusCode.NotFound;
        msg = Encoding.UTF8.GetBytes ("Sorry, that page does not exist");
      }
      else
      {
        context.Response.StatusCode = (int)HttpStatusCode.OK;
        msg = File.ReadAllBytes (path);
      }
      context.Response.ContentLength64 = msg.Length;
      using (Stream s = context.Response.OutputStream)
        await s.WriteAsync (msg, 0, msg.Length);
    }
    catch (Exception ex) { Console.WriteLine ("Request error: " + ex); }
  }
}
static string TempDirectory
{
  get => RuntimeInformation.IsOSPlatform (OSPlatform.Windows) ?
          @"C:\Temp" : "/tmp";
}

DNS

foreach (IPAddress a in Dns.GetHostAddresses ("albahari.com"))
  Console.WriteLine (a.ToString());     // 205.210.42.167

IPHostEntry entry = Dns.GetHostEntry ("205.210.42.167");
Console.WriteLine (entry.HostName);                    // albahari.com

IPAddress address = new IPAddress (new byte[] { 205, 210, 42, 167 });
IPHostEntry entry2 = Dns.GetHostEntry (address);
Console.WriteLine (entry2.HostName);                    // albahari.com

foreach (IPAddress a in await Dns.GetHostAddressesAsync ("albahari.com"))
  Console.WriteLine (a.ToString());

Sending mail with SMTP

var client = new SmtpClient ("smtp.myisp.com", 587)
{
  Credentials = new NetworkCredential ("me@myisp.com", "MySecurePass"),
  EnableSsl = true
};

MailMessage mm = new MailMessage();

mm.Sender = new MailAddress ("kay@domain.com", "Kay");
mm.From = new MailAddress ("kay@domain.com", "Kay");
mm.To.Add (new MailAddress ("bob@domain.com", "Bob"));
mm.CC.Add (new MailAddress ("dan@domain.com", "Dan"));
mm.Subject = "Hello!";
mm.Body = "Hi there. Here's the photo!";
mm.IsBodyHtml = false;
mm.Priority = MailPriority.High;

Attachment a = new Attachment ("photo.jpg",
                               System.Net.Mime.MediaTypeNames.Image.Jpeg);
mm.Attachments.Add (a);
client.Send (mm);

client.Send ("me@myisp.com", "someone@somewhere.com", "Subject", "Body");
Console.WriteLine ("Sent");

TCP - simple demo

class TcpDemo
{
  static void Main()
  {
    new Thread (Server).Start();       // Run server method concurrently.
    Thread.Sleep (500);                // Give server time to start.
    Client();
  }

  static void Client()
  {
    using (TcpClient client = new TcpClient ("localhost", 51111))
    using (NetworkStream n = client.GetStream())
    {
      BinaryWriter w = new BinaryWriter (n);
      w.Write ("Hello");
      w.Flush();
      Console.WriteLine (new BinaryReader (n).ReadString());
    }
  }

  static void Server()     // Handles a single client request, then exits.
  {
    TcpListener listener = new TcpListener (IPAddress.Any, 51111);
    listener.Start();
    using (TcpClient c = listener.AcceptTcpClient())
    using (NetworkStream n = c.GetStream())
    {
      string msg = new BinaryReader (n).ReadString();
      BinaryWriter w = new BinaryWriter (n);
      w.Write (msg + " right back!");
      w.Flush();                      // Must call Flush because we're not
    }                                 // disposing the writer.
    listener.Stop();
  }
}

TCP - Concurrency

void Main()
{
  RunServerAsync();

  using (TcpClient client = new TcpClient ("localhost", 51111))
  using (NetworkStream n = client.GetStream())
  {
    BinaryWriter w = new BinaryWriter (n);
    w.Write (Enumerable.Range (0, 5000).Select (x => (byte) x).ToArray());
    w.Flush();
    Console.WriteLine (new BinaryReader (n).ReadBytes (5000));
  }

}

async void RunServerAsync ()
{
  var listener = new TcpListener (IPAddress.Any, 51111);
  listener.Start ();
  try
  {
    while (true)
      Accept (await listener.AcceptTcpClientAsync ());
  }
  finally { listener.Stop(); }
}

async Task Accept (TcpClient client)
{
  await Task.Yield ();
  try
  {
    using (client)
    using (NetworkStream n = client.GetStream ())
    {
      byte[] data = new byte [5000];

      int bytesRead = 0; int chunkSize = 1;
      while (bytesRead < data.Length && chunkSize > 0)
        bytesRead += chunkSize =
          await n.ReadAsync (data, bytesRead, data.Length - bytesRead);

      Array.Reverse (data);   // Reverse the byte sequence
      await n.WriteAsync (data, 0, data.Length);
    }
  }
  catch (Exception ex) { Console.WriteLine (ex.Message); }
}

Receiving POP3 mail

void Main()
{
  using (TcpClient client = new TcpClient ("mail.isp.com", 110))
  using (NetworkStream n = client.GetStream())
  {
    ReadLine (n);                             // Read the welcome message.
    SendCommand (n, "USER username");
    SendCommand (n, "PASS password");
    SendCommand (n, "LIST");                  // Retrieve message IDs
    List<int> messageIDs = new List<int>();
    while (true)
    {
      string line = ReadLine (n);             // e.g.,  "1 1876"
      if (line == ".") break;
      messageIDs.Add (int.Parse (line.Split (' ') [0]));   // Message ID
    }

    foreach (int id in messageIDs)         // Retrieve each message.
    {
      SendCommand (n, "RETR " + id);
      string randomFile = Guid.NewGuid().ToString() + ".eml";
      using (StreamWriter writer = File.CreateText (randomFile))
        while (true)
        {
          string line = ReadLine (n);      // Read next line of message.
          if (line == ".") break;          // Single dot = end of message.
          if (line == "..") line = ".";    // "Escape out" double dot.
          writer.WriteLine (line);         // Write to output file.
        }
      SendCommand (n, "DELE " + id);       // Delete message off server.
    }
    SendCommand (n, "QUIT");
  }

}

static string ReadLine (Stream s)
{
  List<byte> lineBuffer = new List<byte>();
  while (true)
  {
    int b = s.ReadByte();
    if (b == 10 || b < 0) break;
    if (b != 13) lineBuffer.Add ((byte)b);
  }
  return Encoding.UTF8.GetString (lineBuffer.ToArray());
}

static void SendCommand (Stream stream, string line)
{
  byte[] data = Encoding.UTF8.GetBytes (line + "\r\n");
  stream.Write (data, 0, data.Length);
  string response = ReadLine (stream);
  if (!response.StartsWith ("+OK"))
    throw new Exception ("POP Error: " + response);
}