Remuxed mkv to mp4 raises error on seek

After remuxing from mp4 to mkv and then back again, trying to seek raises an error:

av.error.PermissionError: [Errno 1] Operation not permitted; last error log: [h264] co located POCs unavailable

Also, number of frames is different in the remuxed file. I would not think that is expected as the codec is unchanged?

To reproduce:

import av
import av.datasets

def remux(input_,output):    
    in_stream = input_.streams.video[0]
    out_stream = output.add_stream(template=in_stream)

    for packet in input_.demux(in_stream):
        if packet.dts is None:
            continue
        packet.stream = out_stream
        output.mux(packet)
    input_.close()
    output.close()
    
input_ = av.open(av.datasets.curated("pexels/time-lapse-video-of-night-sky-857195.mp4"))
input_.seek(0)
print(input_.streams[0].frames) #168
output = av.open("remuxed.mkv", "w")
remux(input_, output)

input_=av.open("remuxed.mkv")
output=av.open("remuxed_again.mp4","w")
remux(input_, output)

test_remux=av.open("remuxed_again.mp4")
print(test_remux.streams[0].frames) #166
test_remux.seek(0)