Private Sub RichTextBox1_TextChanged(sender As Object, e As EventArgs) _
  Handles RichTextBox1.TextChanged
 
  Dim rtf As RichTextBox = DirectCast(sender, RichTextBox)
  Dim t, tt As String, a, b As String(), i, min As Short
  Dim c, cm As Color
 
  t = rtf.Rtf
  min = 1
  For Each Item As Match In Regex.Matches(t, "\redd{1,3}\greend{1,3}\blued{1,3}")
    tt += Item.Value.Substring(1) + ";"
  Next
  a = Split(tt, ";")
  For i = 0 To a.Length - 1
    If a(i) <> "" Then
      a(i) = a(i).Replace("red", "").Replace("green", "").Replace("blue", "")
      b = Split(a(i), "")
      c = Color.FromArgb(0, CShort(b(0)), CShort(b(1)), CShort(b(2)))
      If getColorBright(c) > min Then
        min = getColorBright(c)
        cm = c
      End If
    End If
  Next 
  rtf.BackColor = GegenFarbe_sw(cm)
  t = ""
End Sub

Und das sind die beiden verwendeten Grafikfinktionen:

Public Function getColorBright(farbe As Color) As Short
  Dim R As Integer = farbe.R ' Farb-Anteile
  Dim G As Integer = farbe.G
  Dim B As Integer = farbe.B
 
  Return CInt(Math.Sqrt(R * R * 0.241 + G * G * 0.691 + B * B * 0.068))
End Function

Public Function GegenFarbe_sw(farbe As Color) As Color
  Dim R As Integer = farbe.R ' Farb-Anteile
  Dim G As Integer = farbe.G
  Dim B As Integer = farbe.B
 
  R = R * 2 : G = G * 5 : B = B
  If R + G + B > 1024 Then
    Return Color.Black     ' schwarz, weil sehr hell
  Else
    Return Color.White     ' weiss, weil sehr dunkel
  End If
End Function