You can find the ResultBuffer TypedValue related codes in the LispDataType enum. As you can see RTRESBUF/5023 is not in the list. You could place the nested part in a LispDataType.ListBegin/LispDataType.ListEnd section instead:
Code:
[LispFunction("GetNestedList")]
public static ResultBuffer GetNestedList(ResultBuffer resBufIn)
{
ResultBuffer resBufOut = new ResultBuffer();
resBufOut.Add(
new TypedValue((int)LispDataType.Text, "Main List Item 1"));
resBufOut.Add(new TypedValue((int)LispDataType.ListBegin));
resBufOut.Add(
new TypedValue((int)LispDataType.Text, "Nested List Item 1"));
resBufOut.Add(
new TypedValue((int)LispDataType.Text, "Nested List Item 2"));
resBufOut.Add(new TypedValue((int)LispDataType.ListEnd));
resBufOut.Add(
new TypedValue((int)LispDataType.Text, "Main List Item 2"));
return resBufOut;
}