Protobuf-net Deserialize Open Street Maps

Question

필자의 삶에서 protobuf 파일을 Open Street Maps에서 역 직렬화 할 수 없습니다.

다음 추출을 deserialize하려고합니다. http://download.geofabrik.de/osm/north-america/us-northeast.osm.pbf 노드를 얻으려면 http://code.google.com/p/protobuf-net/을 라이브러리로 사용하고 있습니다. 여러 객체를 비 직렬화하려고 시도했지만 모두 null이되었습니다.

프로토 파일은 여기에서 찾을 수 있습니다 : http://trac.openstreetmap.org/browser/applications/utils/export/osm2pgsql/protobuf

어떤 제안?

Answer 1

오른쪽; 문제는이 단지 protobuf되지 않는 것입니다 - 그것은 하이브리드 파일 형식입니다 (defined here것을 내부적으로 다양한 형식들 사이 protobuf를 포함)를 선택적으로 보이지만 그것은 또한 (압축을 통합 나는 '

.. 내가 스펙에서 할 수있는 것을 뽑아 냈다. 여기에 protobuf-net을 사용하여 청크를 처리하는 C# 리더가있다.이 파일을 행복하게 읽으면서 끝까지 읽는다. 나는 4515 블록 (BlockHeader) . Blob에 도착했을 때 나는 spec demarks가 어떻게 되느냐에 대해 다소 혼란 스럽다. 여기서는 제안에 대해 열려있다. 사용중인 zlib 압축을 처리하기 위해 ZLIB.NET도 사용했다. 의 이 문제를 해결하기 위해 ZLIB 데이터를 처리하고 소유권이 주장 된 크기와 비교하여 유효성을 검사하여 적어도 정상적인 지 확인했습니다.

OSMHeader과 OSMData을 구분하는 방법을 알아낼 수 있다면 (또는 작성자에게 질문 할 수 있습니다.) 내가 뭔가 다른 것을 기쁘게 생각합니다. 여기에서 멈춰서도 괜찮을 까 바랍니다.하지만 몇 시간 마가에 의해 윤곽 설정 후 페이지

using System; 
using System.IO; 
using OpenStreetMap; // where my .proto-generated entities are living 
using ProtoBuf; // protobuf-net 
using zlib; // ZLIB.NET  

class OpenStreetMapParser 
{ 

    static void Main() 
    { 
     using (var file = File.OpenRead("us-northeast.osm.pbf")) 
     { 
      // from http://wiki.openstreetmap.org/wiki/ProtocolBufBinary: 
      //A file contains a header followed by a sequence of fileblocks. The design is intended to allow future random-access to the contents of the file and skipping past not-understood or unwanted data. 
      //The format is a repeating sequence of: 
      //int4: length of the BlockHeader message in network byte order 
      //serialized BlockHeader message 
      //serialized Blob message (size is given in the header) 

      int length, blockCount = 0; 
      while (Serializer.TryReadLengthPrefix(file, PrefixStyle.Fixed32, out length)) 
      { 
       // I'm just being lazy and re-using something "close enough" here 
       // note that v2 has a big-endian option, but Fixed32 assumes little-endian - we 
       // actually need the other way around (network byte order): 
       uint len = (uint)length; 
       len = ((len & 0xFF) << 24) | ((len & 0xFF00) << 8) | ((len & 0xFF0000) >> 8) | ((len & 0xFF000000) >> 24); 
       length = (int)len; 

       BlockHeader header; 
       // again, v2 has capped-streams built in, but I'm deliberately 
       // limiting myself to v1 features 
       using (var tmp = new LimitedStream(file, length)) 
       { 
        header = Serializer.Deserialize<BlockHeader>(tmp); 
       } 
       Blob blob; 
       using (var tmp = new LimitedStream(file, header.datasize)) 
       { 
        blob = Serializer.Deserialize<Blob>(tmp); 
       } 
       if(blob.zlib_data == null) throw new NotSupportedException("I'm only handling zlib here!"); 

       using(var ms = new MemoryStream(blob.zlib_data)) 
       using(var zlib = new ZLibStream(ms)) 
       { // at this point I'm very unclear how the OSMHeader and OSMData are packed - it isn't clear 
        // read this to the end, to check we can parse the zlib 
        int payloadLen = 0; 
        while (zlib.ReadByte() >= 0) payloadLen++; 
        if (payloadLen != blob.raw_size) throw new FormatException("Screwed that up..."); 
       } 
       blockCount++; 
       Console.WriteLine("Read block " + blockCount.ToString()); 


      } 
      Console.WriteLine("all done"); 
      Console.ReadLine(); 
     } 
    } 
} 
abstract class InputStream : Stream 
{ 
    protected abstract int ReadNextBlock(byte[] buffer, int offset, int count); 
    public sealed override int Read(byte[] buffer, int offset, int count) 
    { 
     int bytesRead, totalRead = 0; 
     while (count > 0 && (bytesRead = ReadNextBlock(buffer, offset, count)) > 0) 
     { 
      count -= bytesRead; 
      offset += bytesRead; 
      totalRead += bytesRead; 
      pos += bytesRead; 
     } 
     return totalRead; 
    } 
    long pos; 
    public override void Write(byte[] buffer, int offset, int count) 
    { 
     throw new NotImplementedException(); 
    } 
    public override void SetLength(long value) 
    { 
     throw new NotImplementedException(); 
    } 
    public override long Position 
    { 
     get 
     { 
      return pos; 
     } 
     set 
     { 
      if (pos != value) throw new NotImplementedException(); 
     } 
    } 
    public override long Length 
    { 
     get { throw new NotImplementedException(); } 
    } 
    public override void Flush() 
    { 
     throw new NotImplementedException(); 
    } 
    public override bool CanWrite 
    { 
     get { return false; } 
    } 
    public override bool CanRead 
    { 
     get { return true; } 
    } 
    public override bool CanSeek 
    { 
     get { return false; } 
    } 
    public override long Seek(long offset, SeekOrigin origin) 
    { 
     throw new NotImplementedException(); 
    } 
} 
class ZLibStream : InputStream 
{ // uses ZLIB.NET: http://www.componentace.com/download/download.php?editionid=25 
    private ZInputStream reader; // seriously, why isn't this a stream? 
    public ZLibStream(Stream stream) 
    { 
     reader = new ZInputStream(stream); 
    } 
    public override void Close() 
    { 
     reader.Close(); 
     base.Close(); 
    } 
    protected override int ReadNextBlock(byte[] buffer, int offset, int count) 
    { 
     // OMG! reader.Read is the base-stream, reader.read is decompressed! yeuch 
     return reader.read(buffer, offset, count); 
    } 

} 
// deliberately doesn't dispose the base-stream  
class LimitedStream : InputStream 
{ 
    private Stream stream; 
    private long remaining; 
    public LimitedStream(Stream stream, long length) 
    { 
     if (length < 0) throw new ArgumentOutOfRangeException("length"); 
     if (stream == null) throw new ArgumentNullException("stream"); 
     if (!stream.CanRead) throw new ArgumentException("stream"); 
     this.stream = stream; 
     this.remaining = length; 
    } 
    protected override int ReadNextBlock(byte[] buffer, int offset, int count) 
    { 
     if(count > remaining) count = (int)remaining; 
     int bytesRead = stream.Read(buffer, offset, count); 
     if (bytesRead > 0) remaining -= bytesRead; 
     return bytesRead; 
    } 
} 

Answer 2

좀 더 작은 영역을 얻으려고 했습니까? us-pacific.osm.pbf

결국 오류 메시지를 게시하는 것이 유용 할 것입니다.

Answer 3

나는 여기에 최종 코드 http://git.openstreetmap.nl/index.cgi/pbf2osm.git/tree/src/main.c?h=35116112eb0066c7729a963b292faa608ddc8ad7

보고 마지막 부분을 알아 냈어.

using System; 
using System.Diagnostics; 
using System.IO; 
using crosby.binary; 
using OSMPBF; 
using PerlLLC.Tools; 
using ProtoBuf; 
using zlib; 

namespace OpenStreetMapOperations 
{ 
    class OpenStreetMapParser 
    { 
     static void Main() 
     { 
      using (var file = File.OpenRead(StaticTools.AssemblyDirectory + @"\us-pacific.osm.pbf")) 
      { 
       // from http://wiki.openstreetmap.org/wiki/ProtocolBufBinary: 
       //A file contains a header followed by a sequence of fileblocks. The design is intended to allow future random-access to the contents of the file and skipping past not-understood or unwanted data. 
       //The format is a repeating sequence of: 
       //int4: length of the BlockHeader message in network byte order 
       //serialized BlockHeader message 
       //serialized Blob message (size is given in the header) 

       int length, blockCount = 0; 
       while (Serializer.TryReadLengthPrefix(file, PrefixStyle.Fixed32, out length)) 
       { 
        // I'm just being lazy and re-using something "close enough" here 
        // note that v2 has a big-endian option, but Fixed32 assumes little-endian - we 
        // actually need the other way around (network byte order): 
        length = IntLittleEndianToBigEndian((uint)length); 

        BlockHeader header; 
        // again, v2 has capped-streams built in, but I'm deliberately 
        // limiting myself to v1 features 
        using (var tmp = new LimitedStream(file, length)) 
        { 
         header = Serializer.Deserialize<BlockHeader>(tmp); 
        } 
        Blob blob; 
        using (var tmp = new LimitedStream(file, header.datasize)) 
        { 
         blob = Serializer.Deserialize<Blob>(tmp); 
        } 
        if (blob.zlib_data == null) throw new NotSupportedException("I'm only handling zlib here!"); 

        HeaderBlock headerBlock; 
        PrimitiveBlock primitiveBlock; 

        using (var ms = new MemoryStream(blob.zlib_data)) 
        using (var zlib = new ZLibStream(ms)) 
        { 
         if (header.type == "OSMHeader") 
          headerBlock = Serializer.Deserialize<HeaderBlock>(zlib); 

         if (header.type == "OSMData") 
          primitiveBlock = Serializer.Deserialize<PrimitiveBlock>(zlib); 
        } 
        blockCount++; 
        Trace.WriteLine("Read block " + blockCount.ToString()); 


       } 
       Trace.WriteLine("all done"); 
      } 
     } 

     // 4-byte number 
     static int IntLittleEndianToBigEndian(uint i) 
     { 
      return (int)(((i & 0xff) << 24) + ((i & 0xff00) << 8) + ((i & 0xff0000) >> 8) + ((i >> 24) & 0xff)); 
     } 
    } 

    abstract class InputStream : Stream 
    { 
     protected abstract int ReadNextBlock(byte[] buffer, int offset, int count); 
     public sealed override int Read(byte[] buffer, int offset, int count) 
     { 
      int bytesRead, totalRead = 0; 
      while (count > 0 && (bytesRead = ReadNextBlock(buffer, offset, count)) > 0) 
      { 
       count -= bytesRead; 
       offset += bytesRead; 
       totalRead += bytesRead; 
       pos += bytesRead; 
      } 
      return totalRead; 
     } 
     long pos; 
     public override void Write(byte[] buffer, int offset, int count) 
     { 
      throw new NotImplementedException(); 
     } 
     public override void SetLength(long value) 
     { 
      throw new NotImplementedException(); 
     } 
     public override long Position 
     { 
      get 
      { 
       return pos; 
      } 
      set 
      { 
       if (pos != value) throw new NotImplementedException(); 
      } 
     } 
     public override long Length 
     { 
      get { throw new NotImplementedException(); } 
     } 
     public override void Flush() 
     { 
      throw new NotImplementedException(); 
     } 
     public override bool CanWrite 
     { 
      get { return false; } 
     } 
     public override bool CanRead 
     { 
      get { return true; } 
     } 
     public override bool CanSeek 
     { 
      get { return false; } 
     } 
     public override long Seek(long offset, SeekOrigin origin) 
     { 
      throw new NotImplementedException(); 
     } 
    } 
    class ZLibStream : InputStream 
    { // uses ZLIB.NET: http://www.componentace.com/download/download.php?editionid=25 
     private ZInputStream reader; // seriously, why isn't this a stream? 
     public ZLibStream(Stream stream) 
     { 
      reader = new ZInputStream(stream); 
     } 
     public override void Close() 
     { 
      reader.Close(); 
      base.Close(); 
     } 
     protected override int ReadNextBlock(byte[] buffer, int offset, int count) 
     { 
      // OMG! reader.Read is the base-stream, reader.read is decompressed! yeuch 
      return reader.read(buffer, offset, count); 
     } 

    } 
    // deliberately doesn't dispose the base-stream  
    class LimitedStream : InputStream 
    { 
     private Stream stream; 
     private long remaining; 
     public LimitedStream(Stream stream, long length) 
     { 
      if (length < 0) throw new ArgumentOutOfRangeException("length"); 
      if (stream == null) throw new ArgumentNullException("stream"); 
      if (!stream.CanRead) throw new ArgumentException("stream"); 
      this.stream = stream; 
      this.remaining = length; 
     } 
     protected override int ReadNextBlock(byte[] buffer, int offset, int count) 
     { 
      if (count > remaining) count = (int)remaining; 
      int bytesRead = stream.Read(buffer, offset, count); 
      if (bytesRead > 0) remaining -= bytesRead; 
      return bytesRead; 
     } 
    } 
} 

Answer 4

예, 그것은 (코드 아래 .. OSM Fileformat.proto 파일을 기반으로.) Fileformat.cs에 protogen에서 온 여기

package OSM_PROTO; 
    message Blob { 
    optional bytes raw = 1; 
    optional int32 raw_size = 2; 
    optional bytes zlib_data = 3; 
    optional bytes lzma_data = 4; 
    optional bytes bzip2_data = 5; 
    } 

    message BlockHeader { 
    required string type = 1; 
    optional bytes indexdata = 2; 
    required int32 datasize = 3; 
    } 

가 생성 된 파일에 BlockHeader의 선언입니다 :

public sealed partial class BlockHeader : pb::GeneratedMessage<BlockHeader, BlockHeader.Builder> {...} 

-> pb = global :: Google.ProtocolBuffers;

는

(ProtocolBuffers.dll)이 패키지와 함께 제공 :

http://code.google.com/p/protobuf-csharp-port/downloads/detail?name=protobuf-csharp-port-2.4.1.473-full-binaries.zip&can=2&q=