冷知識 - base64url 編碼

寫 Side Project 時踩到一個問題。某物件使用 System.Text.Json 序列化成 JSON，試著用 Json.NET 解析時冒出 "The input is not a valid Base-64 string as it contains a non-base 64 character, more than two padding characters, or a non-white space character among the padding characters" 錯誤。

追進程式碼，發現是物件在 Id 屬性自訂了名為 Base64UrlConverter 的 Json 轉換規則，此一設定只對 System.Text.Json 有效，其轉換結果並非標準 Base64，而 Json.NET 會把它當成 Base64 處理，於是便爆炸了。

public sealed class PublicKeyCredentialDescriptor
{
    public PublicKeyCredentialDescriptor(byte[] id)
        : this(PublicKeyCredentialType.PublicKey, id, null) { }

    [JsonConstructor]
    public PublicKeyCredentialDescriptor(PublicKeyCredentialType type, byte[] id, AuthenticatorTransport[]? transports = null)
    {
        ArgumentNullException.ThrowIfNull(id);

        Type = type;
        Id = id;
        Transports = transports;
    }

    /// <summary>
    /// This member contains the credential ID of the public key credential the caller is referring to.
    /// </summary>
    [JsonConverter(typeof(Base64UrlConverter))]
    [JsonPropertyName("id")]
    public byte[] Id { get; }

    //... 略 ...
};

/// <summary>
/// Custom Converter for encoding/encoding byte[] using Base64Url instead of default Base64.
/// </summary>
public sealed class Base64UrlConverter : JsonConverter<byte[]>
{
    public override byte[] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
    {
        if (!reader.HasValueSequence)
        {
            return Base64Url.DecodeUtf8(reader.ValueSpan);
        }
        else
        {
            return Base64Url.Decode(reader.GetString());
        }
    }

    public override void Write(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options)
    {
        writer.WriteStringValue(Base64Url.Encode(value));
    }
}

/// <summary>
/// Helper class to handle Base64Url. Based on Carbon.Jose source code.
/// </summary>
public static class Base64Url
{
    /// <summary>
    /// Converts arg data to a Base64Url encoded string.
    /// </summary>
    public static string Encode(ReadOnlySpan<byte> arg)
    {
        int base64Length = (int)(((long)arg.Length + 2) / 3 * 4);

        char[] pooledBuffer = ArrayPool<char>.Shared.Rent(base64Length);

        Convert.TryToBase64Chars(arg, pooledBuffer, out int encodedLength);

        Span<char> base64Url = pooledBuffer.AsSpan(0, encodedLength);

        for (int i = 0; i < base64Url.Length; i++)
        {
            ref char c = ref base64Url[i];

            switch (c)
            {
                case '+':
                    c = '-';
                    break;
                case '/':
                    c = '_';
                    break;
            }
        }

        int equalIndex = base64Url.IndexOf('=');

        if (equalIndex > -1) // remove trailing equal characters
        {
            base64Url = base64Url.Slice(0, equalIndex);
        }

        var result = new string(base64Url);

        ArrayPool<char>.Shared.Return(pooledBuffer, clearArray: true);

        return result;
    }

    /// <summary>
    /// Decodes a Base64Url encoded string to its raw bytes.
    /// </summary>
    public static byte[] Decode(ReadOnlySpan<char> text)
    {
        int padCharCount = (text.Length % 4) switch
        {
            2 => 2,
            3 => 1,
            _ => 0
        };

        int encodedLength = text.Length + padCharCount;

        char[] buffer = ArrayPool<char>.Shared.Rent(encodedLength);

        text.CopyTo(buffer);

        for (int i = 0; i < text.Length; i++)
        {
            ref char c = ref buffer[i];

            switch (c)
            {
                case '-':
                    c = '+';
                    break;
                case '_':
                    c = '/';
                    break;
            }
        }

        if (padCharCount == 1)
        {
            buffer[encodedLength - 1] = '=';
        }
        else if (padCharCount == 2)
        {
            buffer[encodedLength - 1] = '=';
            buffer[encodedLength - 2] = '=';
        }

        var result = Convert.FromBase64CharArray(buffer, 0, encodedLength);

        ArrayPool<char>.Shared.Return(buffer, true);

        return result;
    }


    /// <summary>
    /// Decodes a Base64Url encoded string to its raw bytes.
    /// </summary>
    public static byte[] DecodeUtf8(ReadOnlySpan<byte> text)
    {
        int padCharCount = (text.Length % 4) switch
        {
            2 => 2,
            3 => 1,
            _ => 0
        };

        int encodedLength = text.Length + padCharCount;

        byte[] buffer = ArrayPool<byte>.Shared.Rent(encodedLength);

        text.CopyTo(buffer);

        for (int i = 0; i < text.Length; i++)
        {
            ref byte c = ref buffer[i];

            switch ((char)c)
            {
                case '-':
                    c = (byte)'+';
                    break;
                case '_':
                    c = (byte)'/';
                    break;
            }
        }

        if (padCharCount == 1)
        {
            buffer[encodedLength - 1] = (byte)'=';
        }
        else if (padCharCount == 2)
        {
            buffer[encodedLength - 1] = (byte)'=';
            buffer[encodedLength - 2] = (byte)'=';
        }

        if (OperationStatus.Done != Base64.DecodeFromUtf8InPlace(buffer.AsSpan(0, encodedLength), out int decodedLength))
        {
            throw new FormatException("The input is not a valid Base-64 string as it contains a non-base 64 character, more than two padding characters, or an illegal character among the padding characters.");
        }

        var result = buffer.AsSpan(0, decodedLength).ToArray();

        ArrayPool<byte>.Shared.Return(buffer, true);

        return result;
    }
}

查了一下，Base 64 Encoding with URL and Filename Safe Alphabet (以下簡稱 base64url) 也算一種公開編碼規格，被定義在 RFC4648 The Base16, Base32, and Base64 Data Encodings，為 Base64 編碼的變化型，主要用於 URL 和檔案名稱。Base64 使用的 "+" 與 "/" 符號在 URL 具有特殊意義，分別代表空白及路徑分隔符號，寫在 URL 時必須跳脫處理。為解決這個問題，Base64URL 將這些字元替換為 URL 安全的字元，"+" 替換為 "-"（減號），"/" 替換為 "_"（底線），若長度已知，結尾的 "=" 填充字元亦可省略。如此 base64url 編碼可直接用於 URL 和檔案名稱，不需要特別編碼或解碼。

學到一則經驗跟一則新知識：

1. 同一資料物件使用 System.Text.Json 與 Json.NET 轉換的 JSON 有可能不同，關鍵在物件或屬性可能存在只對特定程式庫有意義的專屬 Attribute，導致換結果不同。
2. 除了 Base64 外，還有 Base16、Base32、base64url 等編譯規則，如有特殊需求可應用。

Introduce to the Base64URL encoding.