DEV Community: kyorohiro (kiyohiro kawamura)

DNS Compression In Dart

kyorohiro (kiyohiro kawamura) — Tue, 28 Dec 2021 21:19:07 +0000

Last time, we successfully generated a DNS Query and get the results from the DNS server.

I would like to analyze this result, but this data is compressed.

In this Section. I'll explain about dns compression.

DNS Message Compression

According to (RFC1035)[https://datatracker.ietf.org/doc/html/rfc1035], We can specify the location to be referenced by OFFSET in the following format.

a domain name represented as a sequence of labels, where each label consists of a length octet followed by that number of octets.  

The domain name terminates with the zero length octet for the null label of the root.

+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
| 1  1|                OFFSET                   |
+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+

For example, adding "example.com" to the buffer would look like this

     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  13 |           6           |           e           |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  22 |           x           |           a           |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  24 |           m           |           p           |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  26 |           l           |           e           |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  28 |           3           |           c           |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  30 |           o           |           m           |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  30 |           0           | 
     +--+--+--+--+--+--+--+--+

From now on, if you return a URL named www.example.com, you can use


     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  40 |           3           |           w           |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  42 |           w           |           w           |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  44 | 1  1|                13                       |
     +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+

You will be able to express yourself in the above way.

Write This In Dart

Let's write a code to compress URLs, which can be written in about 30 lines of code.

// dnsdict.dart
import 'dart:convert';
import 'dart:typed_data' show Uint8List;

class DNSCompressionDictItem {
  int index;
}

class DNSCompressionDict {
  Map<String, DNSCompressionDictItem> dict = {};
  Uint8List add(String item, int index) {
    var items = item.split('.');
    var buffer = <int>[];
    for (var i = 0; i < items.length; i++) {
      var key = items.sublist(i).join('.');
      if (dict.containsKey(key)) {
        // 既に登録されていれば、そのアドレスを返す
        var tmp = dict[key].index | 0xC000;
        buffer.addAll([(tmp >> 8) & 0xFF, tmp & 0xFF]);
        return Uint8List.fromList(buffer);
      } else {
        // 登録されていないならば、保存する
        buffer.add(items[i].length);
        buffer.addAll(ascii.encode(items[i]));
        dict[key] = DNSCompressionDictItem()..index = index;
        index += items[i].length + 1;
      }
    }
    if (buffer.isNotEmpty) {
      buffer.add(0);
    }
    // 登録されていないならば、保存する
    return Uint8List.fromList(buffer);
  }
}

// dnsdict_test.dart

import 'package:info.kyorohiro.dns/dns.dart';
import 'package:test/test.dart';

void main() {
  group('DNSName', () {
    setUp(() {});

    test('DNSName.encode()', () {
      var dict = DNSCompressionDict();
      int index = 0;
      {
        var bufferSrc = dict.add('yahoo.co.jp', 0);
        index += bufferSrc.length;
        expect(DNSBuffer.fromList(bufferSrc).toHex(), '057961686f6f02636f026a7000');
      }
      // 057961686f6f02636f026a7000(13)

      {
        var bufferSrc = dict.add('google.co.jp', index);
        index += bufferSrc.length;
        expect(DNSBuffer.fromList(bufferSrc).toHex(), '06676f6f676c65c006');
      }
      // 057961686f6f02636f026a7000(13)
      // 06676f6f676c65c006(9)

      {
        var bufferSrc = dict.add('www.google.co.jp', index);
        index += bufferSrc.length;
        expect(DNSBuffer.fromList(bufferSrc).toHex(), '03777777c00d');
      }
      // 057961686f6f02636f026a7000(13)
      // 06676f6f676c65c006(9)
      // 03777777c00d(6)

      {
        var bufferSrc = dict.add('www.google.co.jp', index);
        index += bufferSrc.length;
        expect(DNSBuffer.fromList(bufferSrc).toHex(), 'c016');
      }
    });
  });
}

Decompress In Dart

If you are writing in C, you need to check if you are accessing invalid memory. However, since this is a Dart program, we have not checked for infinite loops.
You may want to check for infinite loops.

This too can be written in about 30 lines of code.

// dnsname.dart

  static Tuple2<String, int> createUrlFromName(Uint8List srcBuffer, int index) {
    var outBuffer = StringBuffer();
    var i = index;
    for (; i < srcBuffer.length;) {
      var nameLength = srcBuffer[i];
      if (nameLength == 0) {
        // TEXT END
        i++;
        return Tuple2<String, int>(outBuffer.toString(), i - index);
      } else if ((0xC0 & nameLength) == 0xC0) {
        // Compression
        var v = ((nameLength & 0x3f) << 8) | srcBuffer[++i];
        var r = createUrlFromName(srcBuffer, v);
        if (outBuffer.length > 0) {
          outBuffer.write('.');
        }
        outBuffer.write(r.item1);
        i++;
        return Tuple2<String, int>(outBuffer.toString(), i - index);
      } else {
        var nameBytes = srcBuffer.sublist(i + 1, i + 1 + nameLength);
        if (outBuffer.length > 0) {
          outBuffer.write('.');
        }
        outBuffer.write(ascii.decode(nameBytes, allowInvalid: true));
        i = i + 1 + nameLength;
      }
    }
    throw DNSNameException('Not Found Null Char');
  }

Next time

Parse the DNS Message retrieved from the DNS server last time and display the result.

Create DNS Query In Dart Language

kyorohiro (kiyohiro kawamura) — Sun, 26 Dec 2021 14:36:31 +0000

In this Section, I will explain how to create DNS Query for getting a A Record.

And, in this section, We will try to use Google's Public DNS.
By sending a Get request to

https://dns.google/dns-query?dns=${base64ized DNS Message}

You can retrieve a record from DNS by sending a Get request to

DNS Message Format

The DNS format consists of five sections: Header, Question, Answer, Authority, Additional.

4. MESSAGES

4.1. Format

    +---------------------+
    |        Header       |
    +---------------------+
    |       Question      | the question for the name server
    +---------------------+
    |        Answer       | RRs answering the question
    +---------------------+
    |      Authority      | RRs pointing toward an authority
    +---------------------+
    |      Additional     | RRs holding additional information
    +---------------------+

from https://datatracker.ietf.org/doc/html/rfc1035

Header for Request

Header Section is following format.

4.1.1. Header section format
                                    1  1  1  1  1  1
      0  1  2  3  4  5  6  7  8  9  0  1  2  3  4  5
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |                      ID                       |
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |QR|   Opcode  |AA|TC|RD|RA|   Z    |   RCODE   |
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |                    QDCOUNT                    |
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |                    ANCOUNT                    |
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |                    NSCOUNT                    |
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |                    ARCOUNT                    |
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+

But if you just want to get A RECORD, you can do the following

ID can be anything.
RD will be 1.
QDCOUNT will be 1.
Otherwise, it can be 0.

Let's write this at dart.


var buffer = DNSBuffer(12);

void setInt16AtBE(Uint8List _buffer, int index, int value) {
  _buffer[index + 0] = (value >> 8) & 0xFF;
  _buffer[index + 1] = (value >> 0) & 0xFF;
}

void main() {
  var buffer = Uint8List(12);
  for (var i = 0; i < 12; i++) {
    buffer[i] = 0;
  }
  setInt16AtBE(buffer, 0, 0x1234);
  buffer[2] = 0x01;
  setInt16AtBE(buffer, 5, 0x01);
  print(toHex(buffer)); // 123401000001000000000000
}

Question for Request

                                    1  1  1  1  1  1
      0  1  2  3  4  5  6  7  8  9  0  1  2  3  4  5
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |                                               |
    /                     QNAME                     /
    /                                               /
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |                     QTYPE                     |
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    |                     QCLASS                    |
    +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+

The format of the Question's Section is as above, and for QNAME If you want to get A RECORD,

QTYPE should be set to 1
QCLASS should be set to 1

QNAME should be set to the domain name, not a string like github.com.

QNAME

       +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    12 |           6           |           g           |
       +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    14 |           i           |           t           |
       +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    16 |           h           |           u           |
       +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    26 |           b           |           3           |
       +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    28 |           c           |           o           |
       +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
    30 |           m           |           0           |
       +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+

The above is a single-byte number, an ASCII string, a single-byte number, and an ASCII string.

The Null character is added at the end.

Let's write this at dart.


void main() {
  var host = 'github.com';
  var splitHost = host.split('.');
  // Calc Buffer Size
  var length = splitHost.length;
  splitHost.forEach((e) {
    length += e.length;
  });
  length += 1; // NULL CHAR
  length += 4; // CLASS  AND TYPE

  // Set Value
  var buffer = Uint8List(length);
  for (var i = 0; i < 12; i++) {
    buffer[i] = 0;
  }

  var index = 0;
  splitHost.forEach((e) {
    buffer[index++] = e.length;
    for (var i = 0; i < e.length; i++) {
      buffer[index++] = ascii.encode(e.substring(i, i + 1))[0];
    }
  });

  setInt16AtBE(buffer, length - 4, 0x01);
  setInt16AtBE(buffer, length - 2, 0x01);
  print(toHex(buffer)); // 0667697468756203636f6d0000010001
}

Request at Browser

If you combine the Header and Question, you get 1234010000010000000000000667697468756203636f6d0000010001 byte data. When converted to Base64,

import 'dart:typed_data' show Uint8List;
import 'dart:convert' show base64;

Uint8List fromHexString(String hexSrc) {
  var _buffer = Uint8List(hexSrc.length ~/ 2);
  for (var i = 0, j = 0; i < hexSrc.length; i += 2, j++) {
    var v = int.parse(hexSrc.substring(i, i + 2), radix: 16);
    _buffer[j] = v & 0xFF;
  }
  return _buffer;
}

void main() {
  var buffer = fromHexString('1234010000010000000000000667697468756203636f6d0000010001');
  print(base64.encode(buffer)); // EjQBAAABAAAAAAAABmdpdGh1YgNjb20AAAEAAQ==
}

it becomes EjQBAAABAAAAAAAABmdpdGh1YgNjb20AAAEAAQ==

From this, if we remove == and apply it to the Google Public DNS address, we get the following

https://dns.google/dns-query?dns=EjQBAAABAAAAAAAABmdpdGh1YgNjb20AAAEAAQ

You can get following DNS Response

1234818000010001000000000667697468756203636f6d0000010001c00c000100010000003c000434c04859

REF

https://github.com/kyorohiro/dart2.dns

https://datatracker.ietf.org/doc/html/rfc1035

Dart's Network Byte Order

kyorohiro (kiyohiro kawamura) — Sun, 26 Dec 2021 13:34:31 +0000

This section explains network byte orders.
There is no special problem when storing one byte of data in memory.

import 'dart:typed_data' show Uint8List;

String toHex(Uint8List buffer) {
  const List<String> vv = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'];
    var b = StringBuffer();
    for (var i = 0; i < buffer.length; i++) {
      var v = buffer[i];
      var v1 = (v >> 4) & 0xF;
      var v2 = v & 0xF;
      b.write('${vv[v1]}${vv[v2]}');
    }
    return b.toString();
}

main() async {
  var input = <int>[0x00, 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f];  
  var buffer = Uint8List(16);
  for(var i=0;i<input.length;i++){
    buffer[i] = input[i] & 0xFF;
  }
  print(toHex(buffer));
}

When this is executed, it will output data such as 000102030405060708090a0b0c0d0e0f.

However, the problem is how to record 2 bytes of short data, 4 bytes of int data
etc. should be recorded in memory...

There are several recording methods, but two are the most common.

Big Endian
When 0x1234 is recorded in memory, it is recorded as [0x12,0x34].
Little endian
When 0x1234 is recorded in memory, it is recorded as [0x34,0x12].

Try to store it in little-endian format

main() async {
  var input = 0x1234;
  var buffer = Uint8List(2);

  //print(((input>>8) & 0xFF).toRadixString(16)); --> 12
  //print((input & 0xFF).toRadixString(16)); --> 34
  buffer[0] = input & 0xFF;
  buffer[1] = (input>>8) & 0xFF;
  print(toHex(buffer));
}

If it is little-endian, it should be displayed in the order 0x34 0x12.
If it is big-endian, it should display 0x12 0x32 in that order.

When we run this, it returns 3412, so we have successfully stored the data as little-endian.

Try to store it in big-endian format


main() async {
  var input = 0x1234;
  var buffer = Uint8List(2);

  //print(((input>>8) & 0xFF).toRadixString(16)); --> 12
  // print((input & 0xFF).toRadixString(16)); --> 34
  buffer[0] = (input>>8) & 0xFF;
  buffer[1] = input & 0xFF;
  print(toHex(buffer));
}

This will return 1234, so we have successfully stored the data in big-endian format.

Communication Protocols are Big Endian

In this article, we will implement DNS which is described in RFCs. In these protocols, data is exchanged in big-endian unless otherwise stated.

How to create DNS Client at Pure Dart

kyorohiro (kiyohiro kawamura) — Sun, 26 Dec 2021 12:39:06 +0000

In this series, I'll show you how to create a DNS client from scratch in Dart language.

(1) Dart's Network Byte Order (ASIS OK)
(2) Generate DNS Query (ASIS OK)
(3) PARSE RESPONSE MESSAGE (TODO)
(4) ...

As a side effect.
Maybe you will be able to read and implement the RFC.

DNS RFC

https://datatracker.ietf.org/doc/html/rfc1034
https://datatracker.ietf.org/doc/html/rfc1035
https://datatracker.ietf.org/doc/html/rfc3901
https://datatracker.ietf.org/doc/html/rfc8484

COMPLETE CODE

https://github.com/kyorohiro/dart2.dns

Dart Bencode Package

kyorohiro (kiyohiro kawamura) — Thu, 08 Apr 2021 00:19:30 +0000

Update Dart Bencode Package

https://github.com/kyorohiro/dart2.bencode

support SDK
- dart:io
- dart:html

sdk: '>=2.12.0 <3.0.0'

for dart v1 is here https://github.com/kyorohiro/dart.bencode

Usage

dependencies:
  info.kyorohiro.dart2.bencode:
    git:
      url: https://github.com/kyorohiro/dart.tiny_parser.git
      ref: v0.0.2

import 'package:info.kyorohiro.dart2.bencode/bencode.dart';
import 'package:dart2/dart2.dart';

{
  var out = Bencode.encode(1024);
  unit.expect('i1024e', convert.utf8.decode(out.toList()));
  num ret = Bencode.decode(out);
  unit.expect(1024, ret);
}

{
  var out = Bencode.encode('test');
  unit.expect('4:test', convert.utf8.decode(out.toList()));
  var text = Bencode.decode(out) as type.Uint8List;
  unit.expect('test', convert.utf8.decode(text.toList()));
}

{
  var l = [];
  l.add('test');
  l.add(1024);
  var out = Bencode.encode(l);
  unit.expect('l4:testi1024ee', convert.utf8.decode(out.toList()));

  List list = Bencode.decode(out);
  unit.expect('test', convert.utf8.decode(list[0].toList()));
  unit.expect(1024, list[1]);
}

{
  var m = <String, Object>{};
  m['test'] = 'test';
  m['value'] = 1024;
  var out = Bencode.encode(m);
  unit.expect('d4:test4:test5:valuei1024ee', convert.utf8.decode(out.toList()));

  Map me = Bencode.decode(out);
  unit.expect('test', convert.utf8.decode(me['test'].toList()));
  unit.expect(1024, me['value']);
}

What is Bencode

B-encode is the encoding for BitTorrent network.
support following type of values.

byte strings
integers
lists
dictionaries

Bencode's unique point compare to json format is to use bytes data.

https://en.wikipedia.org/wiki/Bencode

Isolate and Dart Http Server

kyorohiro (kiyohiro kawamura) — Sun, 04 Apr 2021 18:34:31 +0000

I have explained various things so far. how to start http server and to start https server and to get SSL and to run isolate.

In this time, I want to explain, how to run http server with isolate.

Hello Isolae Http Server

But, it is not so difficult, because Dart 's HttpServer class support isolate.

you can do it just by seting shared option is true,

import 'dart:io' as io;
import 'dart:isolate' as iso;

onIsolateMain(message) async {
  var server = await io.HttpServer.bind("0.0.0.0", 8080, shared: true);
  await for (var request in server) {
    print("${request.uri}");
    request.response.write("${message}");
    request.response.close();
  }
}

main() async {
  var server = await io.HttpServer.bind("0.0.0.0", 8080, shared: true);
  server.listen((request) {
    print("${request.uri}");
    request.response.write("parent");
    request.response.close();
  });
  //
  for (int i = 0; i < 10; i++) {
    iso.Isolate.spawn(onIsolateMain, "${i}");
  }
}

$ curl http://tetorica.net:8080/
parent
$ curl http://tetorica.net:8080/
1
$ curl http://tetorica.net:8080/
0
$ curl http://tetorica.net:8080/
4

Dart's HttpServer distribute access to server object.

About Dart's Isolate (1)

kyorohiro (kiyohiro kawamura) — Sun, 04 Apr 2021 17:39:09 +0000

I want to explain about How to distribute the load with Isolate.
And In this Document, I'll confirm about isolate basic usage before explain it.

Hello Isolate

We can write multi-threads like programming code to use Isolate.

import 'dart:isolate' as iso;

onMain(message) async {
  //
  print("child:arg:${message}");
  for (var i = 0; i < 5; i++) {
    await Future.delayed(Duration(milliseconds: 100));
    print("child:print:${i}");
  }
}

main() async {
  iso.Isolate.spawn(onMain, "Hi");
  for (var i = 0; i < 5; i++) {
    await Future.delayed(Duration(milliseconds: 100));
    print("parent:print:${i}");
  }
}

$ dart bin/main_example_isolate_01.dart 
child:arg:Hi
parent:print:0
child:print:0
parent:print:1
child:print:1
parent:print:2
child:print:2
parent:print:3
child:print:3
parent:print:4

code https://github.com/kyorohiro/hao_dart_server_and_systemd/blob/master/bin/main_example_isolate_01.dart

Communication For Each Isolate

Parent Isolate and Child Isolate can not share memory.
if you want to share data both isolate, you must to use SendPort and ReceivePort.

import 'dart:isolate' as iso;

onMain(message) async {
  //
  print("child:arg:${message}");
  iso.SendPort sendPort = message['p'];
  for (var i = 0; i < 5; i++) {
    await Future.delayed(Duration(milliseconds: 100));
    print("child:print:${i}");
    sendPort.send("hi${i}");
  }
}

main() async {
  iso.ReceivePort receivePort = iso.ReceivePort();
  receivePort.listen((message) {
    print("parent:onMessage: ${message}");
  });
  iso.Isolate.spawn(onMain, {"v": "Hi", "p": receivePort.sendPort});
  for (var i = 0; i < 5; i++) {
    await Future.delayed(Duration(milliseconds: 100));
    print("parent:print:${i}");
  }
  receivePort.close();
}

$ dart bin/main_example_isolate_02.dart 
child:arg:{v: Hi, p: SendPort}
parent:print:0
child:print:0
parent:onMessage: hi0
parent:print:1
child:print:1
parent:onMessage: hi1
parent:print:2
child:print:2
parent:onMessage: hi2
parent:print:3
child:print:3
parent:onMessage: hi3
parent:print:4
child:print:4
parent:onMessage: hi4

code https://github.com/kyorohiro/hao_dart_server_and_systemd/blob/master/bin/main_example_isolate_02.dart

More Isolate
https://github.com/kyorohiro/hao_dart_server_and_systemd/blob/master/bin/main_example_isolate.dart

Dart HttpServer With SSL

kyorohiro (kiyohiro kawamura) — Sun, 04 Apr 2021 00:42:24 +0000

Last Time I got ssl from let's encrypt.

This time, execute a dart's https server at use ssl cert gotted last time.

import 'dart:io' as io;

const String cerbotWebRootPath = "/var/www/html";
const String privkeyPath = "/etc/letsencrypt/live/tetorica.net/privkey.pem";
const String fullchainPath = "/etc/letsencrypt/live/tetorica.net/fullchain.pem";
void main(List<String> arguments) async {
  try {
    print("start bind");
    onRequest(io.HttpRequest request) async {
      try {
        print("receive requested ${request.uri}");
        if (request.uri.path.startsWith("/.well-known/")) {
          var acmeChallengeFilePath = "" +
              cerbotWebRootPath +
              request.uri.path.replaceAll(RegExp("\\?.*"), "");
          acmeChallengeFilePath = acmeChallengeFilePath.replaceAll("/..", "/");
          var acmeChallengeFile = io.File(acmeChallengeFilePath);
          var acmeChallengeData = await acmeChallengeFile.readAsString();
          request.response.write(acmeChallengeData);
          request.response.close();
        }
        request.response.write("Hello");
        request.response.close();
      } catch (e, s) {
        print("${e}");
        print("${s}");
      }
    }

    var httpServer = await io.HttpServer.bind("0.0.0.0", 80);
    print("binded 80");
    httpServer.listen((request) {
      onRequest(request);
    });

    String key = io.Platform.script.resolve(privkeyPath).toFilePath();
    String crt = io.Platform.script.resolve(fullchainPath).toFilePath();
    io.SecurityContext context = new io.SecurityContext();
    context.useCertificateChain(crt);
    context.usePrivateKey(key, password: "");
    var httpsServer = await io.HttpServer.bindSecure("0.0.0.0", 443, context);
    print("binded 443");
    httpsServer.listen((request) {
      onRequest(request);
    });
  } catch (e, s) {
    print("${e}");
    print("${s}");
  }
}

build and restart.

$ dart2native ./bin/main.dart
$ mv bin/main.exe /opt/main.exe
$ systemctl restart darthelloserver

and check from browser

https://tetorica.net/

Update ssl

check cron deamond

$ systemctl list-unit-files | grep cron
cron.service                           enabled         enabled

setup cron

$ crontab -u root -e

00 04 03 * * certbot renew && systemctl restart darthelloserver

all code

https://github.com/kyorohiro/hao_dart_server_and_systemd/tree/dev03

Dart's Http Server and Isolate

Get SSL Cert At Let's Encrypt

kyorohiro (kiyohiro kawamura) — Sun, 04 Apr 2021 00:16:24 +0000

In the previous document I created an Http Server without SSL.

In this time, I explain to how to get ssl cert from let's encrypt.

Install certbot

I used certbot to communicate Let'sEncrypt.

$ apt-get install certbot -y

HttpServer For Certbot

Certbot's generate a file for acme-challege under '${WebRoot}/.well-known/acme-challenge/' folder.

and get request 'http://${HOST}/.well-known/acme-challenge/xxx'

bin/main.dart

Write a Code at Dart.

import 'dart:io' as io;

const String cerbotWebRootPath = "/var/www/html";

void main(List<String> arguments) async {
  try {
    print("start bind");
    var httpServer = await io.HttpServer.bind("0.0.0.0", 80);
    print("binded");
    await for (var request in httpServer) {
      try {
        print("receive requested ${request.uri}");
        if (request.uri.path.startsWith("/.well-known/")) {
          var acmeChallengeFilePath = "" +
              cerbotWebRootPath +
              request.uri.path.replaceAll(RegExp("\\?.*"), "");
          acmeChallengeFilePath = acmeChallengeFilePath.replaceAll("/..", "/");
          var acmeChallengeFile = io.File(acmeChallengeFilePath);
          var acmeChallengeData = await acmeChallengeFile.readAsString();
          request.response.write(acmeChallengeData);
          request.response.close();
        }
        request.response.write("Hello");
        request.response.close();
      } catch (e, s) {
        print("${e}");
        print("${s}");
      }
    }
  } catch (e, s) {
    print("${e}");
    print("${s}");
  }
}

update server

$ dart2native ./bin/main.dart
$ mv bin/main.exe /opt/main.exe
$ systemctl restart darthelloserver

and, execute certbot at webroot mode

$ certbot certonly --webroot -w /var/www/html -d tetorica.net -m kyorohiro@gmail.com
Saving debug log to /var/log/letsencrypt/letsencrypt.log
Plugins selected: Authenticator webroot, Installer None
Obtaining a new certificate
Performing the following challenges:
http-01 challenge for tetorica.net
Using the webroot path /var/www/html for all unmatched domains.
Waiting for verification...
Cleaning up challenges

IMPORTANT NOTES:
 - Congratulations! Your certificate and chain have been saved at:
   /etc/letsencrypt/live/tetorica.net/fullchain.pem
   Your key file has been saved at:
   /etc/letsencrypt/live/tetorica.net/privkey.pem
   Your cert will expire on 2021-07-02. To obtain a new or tweaked
   version of this certificate in the future, simply run certbot
   again. To non-interactively renew *all* of your certificates, run
   "certbot renew"
 - If you like Certbot, please consider supporting our work by:

   Donating to ISRG / Let's Encrypt:   https://letsencrypt.org/donate
   Donating to EFF:                    https://eff.org/donate-le

all code

https://github.com/kyorohiro/hao_dart_server_and_systemd/tree/dev02

Dart HttpServer & Systemd

kyorohiro (kiyohiro kawamura) — Sat, 03 Apr 2021 18:13:50 +0000

How to setup dart's http server.
Dartlang support httpserver's function at dart:io.

like this.

./bin/main.dart

import 'dart:io' as io;

void main(List<String> arguments) async {
  try {
    print("start bind");
    var httpServer = await io.HttpServer.bind("0.0.0.0", 80);
    print("binded");
    await for (var request in httpServer) {
      print("receive requested ${request.uri}");
      request.response.write("Hello");
      request.response.close();
    }
  } catch (e, s) {
    print("${e}");
    print("${s}");
  }
}

but, if you want to use this code. you must to know "how to setup this server on you linux".

In this serises, I'll explain about systemd, let's encrypt for dart server.

Environment

ubuntu 20.04

Firewall

First of all. setup a firewall.

$ ufw status
Status: inactive
# $ ufw disable 
$ ufw default deny
$ ufw allow 22/tcp
$ ufw allow 80/tcp
$ ufw allow 443/tcp
$ ufw enable
$ ufw status

port 22 is for ssh.you must to allow port 22 if you can access a linux instance to use ssh.

Setup for Systemd

build dart code to native.

$ dart2native bin/main.dart 
$ mv bin/main.exe /opt/main.exe

and create entry shell.

./darthelloserver.sh

#!/bin/sh

#cd /app/hao_dart_server_and_systemd; dart bin/main.dart
/opt/main.exe

and write service setting

./darthelloserver.service

[Unit]
Description=Dart Hello Http Server
After=syslog.target network-online.target

[Service]
ExecStart = /opt/darthelloserver.sh
Restart = always
Type = simple

[Install]
WantedBy=multi-user.target
WantedBy=network-online.target

and regist this service setting to systemd.

$ cp darthelloserver.sh /opt/darthelloserver.sh
$ chmod 655 /opt/darthelloserver.sh
$ cp darthelloserver.service /etc/systemd/system/darthelloserver.service
$ systemctl enable darthelloserver
$ systemctl start darthelloserver

This deamon use systemd-networkd-wait-online.
if systemd-networkd-wait-online is disable status, this deamin didn't run.

you can check like this.

$ systemctl list-unit-files | grep network
$ systemctl enable systemd-networkd
$ systemctl enable systemd-networkd-wait-online

all code

https://github.com/kyorohiro/hao_dart_server_and_systemd/tree/dev01

Code-Server with Dart and Flutter Web On VPS

kyorohiro (kiyohiro kawamura) — Sat, 13 Feb 2021 15:04:18 +0000

In this section, I explain How to create a Develop environment for flutter for web on cloud.

and,
you can use vscode plugin too.

Run Code-server on vps

$ apt-get update
$ curl -fsSL https://code-server.dev/install.sh | sh
$ systemctl enable --now code-server@root
// Created symlink /etc/systemd/system/default.target.wants/code-server@root.service → /lib/systemd/system/code-server@.service.

Setup password (Optional)

$ vi /root/.config/code-server/config.yaml

bind-addr: 0.0.0.0:8080
auth: password
password: MyFlutter-0213f
cert: false

$ systemctl restart code-server@root

Install Flutter

https://flutter.dev/docs/get-started/install/linux

$ apt-get install curl -y
$ apt-get install git -y
$ apt-get install unzip -y
$ apt-get install zip -y
$ git clone https://github.com/flutter/flutter.git
$ export PATH="$PATH:`pwd`/flutter/bin"
$ flutter doctor

Install Flutter Web

https://flutter.dev/docs/get-started/web

$ flutter channel beta
$ flutter upgrade
$ flutter config --enable-web
$ flutter devices

Run Sample Program

$ flutter create myapp
$ cd myapp
$ flutter build web
$ flutter run -d web-server  --web-port=18080 --web-hostname=0.0.0.0

Hello Libtorrent : Extension Peer Plugin

kyorohiro (kiyohiro kawamura) — Thu, 04 Feb 2021 14:33:10 +0000

Last time, I introduced the torrent plugin. This time I will introduce the Peer Plugin.

By using Peer Plugin, you can monitor the interaction between Peers.

Various communications are taking place between peers.

For Example Seeder is

Explorer Peer with Tracker and DHT
Connect Founded Peer
Shakehand between peers
Send Bitfiled Format Data About Piece Info
Receive Request
Response This Request
..
..

You can hook these events by using the Peer Plugin.

struct TORRENT_EXPORT peer_plugin
{
    virtual void add_handshake(...
    virtual void on_disconnect(...
    virtual void on_connected(...
    virtual bool on_handshake(...
    virtual bool on_extension_handshake(...
    virtual bool on_choke(...
    virtual bool on_unchoke(...
    virtual bool on_interested(...
    virtual bool on_not_interested(...
    virtual bool on_have(...
    virtual bool on_dont_have(...
    virtual bool on_bitfield(...
    virtual bool on_have_all(...
    virtual bool on_have_none(...
    virtual bool on_allowed_fast(...
    virtual bool on_request(...
    virtual bool on_piece(...
    virtual bool on_cancel(...
    virtual bool on_reject(...
    virtual bool on_suggest(...
    virtual void sent_have_all(...
    virtual void sent_have_none(...
    virtual void sent_reject_request(...
    virtual void sent_allow_fast(...
    virtual void sent_suggest(...
    virtual void sent_cancel(...
    virtual void sent_request(...
    virtual void sent_choke(...
    virtual void sent_unchoke(...
    virtual void sent_interested(...
    virtual void sent_not_interested(...
    virtual void sent_have(...
    virtual void sent_piece(...
    virtual void sent_payload(...
    virtual bool can_disconnect(...
    virtual bool on_extended(...
    virtual bool on_unknown_message(...
    virtual void on_piece_pass(...
    virtual void on_piece_failed(...
    virtual void tick(...
    virtual bool write_request(...
};

Sample Code

#include <iostream>
#include <libtorrent/session.hpp>
#include <libtorrent/alert_types.hpp>
#include <libtorrent/alert.hpp>
#include <libtorrent/magnet_uri.hpp>
#include <libtorrent/error_code.hpp>
#include <libtorrent/peer_connection_handle.hpp>

// ---
struct test_peer_plugin : lt::peer_plugin
{
    std::string _id = "";
    test_peer_plugin(std::string id)
    {
        _id = id;
    }

    void add_handshake(lt::entry &e) override
    {
        std::cout << _id << " add_handshak " << e.to_string() << std::endl;
    }
    bool on_have(lt::piece_index_t i) override
    {
        std::cout << _id << " on_have " << i << std::endl;
        return false;
    }
    bool on_dont_have(lt::piece_index_t i) override
    {
        std::cout << _id << " on_dont_have " << i << std::endl;
        return false;
    }
    bool on_bitfield(lt::bitfield const &bitfield) override
    {
        std::cout << _id << " on_bitfield " << bitfield.count() << "/" << bitfield.size() << std::endl;
        return false;
    }
    bool on_have_all() override
    {
        std::cout << _id << " on_have_all " << std::endl;
        return false;
    }
    bool on_have_none() override
    {
        std::cout << _id << " on_have_none " << std::endl;
        return false;
    }
    void on_disconnect(lt::error_code const &e) override
    {
        std::cout << _id << " on_disconnect " << e.message() << std::endl;
    }

    void on_connected() override
    {
        std::cout << _id << " on_connect " << std::endl;
    }
};
struct test_plugin_torrent : lt::torrent_plugin
{
    std::shared_ptr<lt::peer_plugin> new_connection(lt::peer_connection_handle const &pch) override
    {
        std::stringstream ss;
        ss << "[remote_ip]=" << pch.remote().address().to_string() << ":" << pch.remote().port();
        ss << "[local_ip]=" << pch.local_endpoint().address().to_string() << ":" << pch.local_endpoint().port();
        std::cout << " new connection 1 " << ss.str() << std::endl;
        return std::make_shared<test_peer_plugin>(ss.str());
    }

    void on_add_peer(lt::tcp::endpoint const &endpoint, lt::peer_source_flags_t flag1, lt::add_peer_flags_t flag2) override
    {
        //using peer_source_flags_t = flags::bitfield_flag<std::uint8_t, struct peer_source_flags_tag>;
        //using alert_category_t    = flags::bitfield_flag<std::uint32_t, struct alert_category_tag>;
        std::string type = "other";
        if (lt::peer_info::dht == flag1)
        {
            type = "dht";
        }
        else if (lt::peer_info::tracker == flag1)
        {
            type = "tracker";
        }
        else if (lt::peer_info::lsd == flag1)
        {
            type = "lsd";
        }
        std::cout << "[on_add_peer][" << type << "]" << endpoint.address().to_string() << ":" << endpoint.port() << std::endl;
    }
};

struct plugin_creator
{
    std::shared_ptr<lt::torrent_plugin>
    operator()(lt::torrent_handle const &, lt::client_data_t)
    {
        return std::make_shared<test_plugin_torrent>();
    }
};
// ----

std::string my_listen_interfaces = ""; // "0.0.0.0:6881,[::]:6881"  "{E4F0B674-0DFC-48BB-98A5-2AA730BDB6D6}:7777"
std::string target_torrentfile_path = "a.torrent";

int main(int argc, char *argv[])
try
{

    //
    // createsession
    lt::settings_pack session_params;
    session_params.set_int(lt::settings_pack::alert_mask, lt::alert_category::all);
    if (my_listen_interfaces.length() != 0)
    {
        session_params.set_str(lt::settings_pack::listen_interfaces, my_listen_interfaces);
    }
    lt::session session(session_params);

    //
    // add torrent
    lt::add_torrent_params torrent_params;
    torrent_params.save_path = ".data"; // save in this dir
    torrent_params.ti = std::make_shared<lt::torrent_info>(target_torrentfile_path);
    plugin_creator creator;

    // ----
    torrent_params.extensions.push_back(creator);
    // ----

    lt::torrent_handle h = session.add_torrent(std::move(torrent_params));

    session.add_torrent(torrent_params);

    while (true)
    {
        std::vector<lt::alert *> alerts;
        session.pop_alerts(&alerts);

        lt::state_update_alert *st;
        for (lt::alert *a : alerts)
        {
            //std::cout << "[" << a->type() << "](" << a->what() << ") " << a->message() << std::endl;
            switch (a->type())
            {
                //case lt::torrent_finished_alert::alert_type:
                //   goto END;
                //case lt::torrent_error_alert::alert_type:
                //    goto END;
            }
        }
        std::this_thread::sleep_for(std::chrono::milliseconds(1000));
    }
END:
    return 0;
}
catch (std::exception &e)
{
    std::cerr << "Error: " << e.what() << std::endl;
}

It looks like a long code, but I've only added the following from the last time.

Write Your Peer Plugin

You can create your peer plugin by extending the peer_plugin object.

struct test_peer_plugin : lt::peer_plugin
{
    std::string _id = "";
    test_peer_plugin(std::string id)
    {
        _id = id;
    }

    void add_handshake(lt::entry &e) override
    {
        std::cout << _id << " add_handshak " << e.to_string() << std::endl;
    }
    bool on_have(lt::piece_index_t i) override
    {
        std::cout << _id << " on_have " << i << std::endl;
        return false;
    }
    bool on_dont_have(lt::piece_index_t i) override
    {
        std::cout << _id << " on_dont_have " << i << std::endl;
        return false;
    }
    bool on_bitfield(lt::bitfield const &bitfield) override
    {
        std::cout << _id << " on_bitfield " << bitfield.count() << "/" << bitfield.size() << std::endl;
        return false;
    }
    bool on_have_all() override
    {
        std::cout << _id << " on_have_all " << std::endl;
        return false;
    }
    bool on_have_none() override
    {
        std::cout << _id << " on_have_none " << std::endl;
        return false;
    }
    void on_disconnect(lt::error_code const &e) override
    {
        std::cout << _id << " on_disconnect " << e.message() << std::endl;
    }

    void on_connected() override
    {
        std::cout << _id << " on_connect " << std::endl;
    }
};

Regist Peer Plugin

You can regist peer plugin from torrent plugin 's new_connection method.


struct test_plugin_torrent : lt::torrent_plugin
{
    std::shared_ptr<lt::peer_plugin> new_connection(lt::peer_connection_handle const &pch) override
    {
        std::stringstream ss;
        ss << "[remote_ip]=" << pch.remote().address().to_string() << ":" << pch.remote().port();
        ss << "[local_ip]=" << pch.local_endpoint().address().to_string() << ":" << pch.local_endpoint().port();
        std::cout << " new connection 1 " << ss.str() << std::endl;
        return std::make_shared<test_peer_plugin>(ss.str());
    }
..
}

Source

https://github.com/kyorohiro/hello_libtorrent
app/main_extension_peer_plugin.cpp