Page MenuHomec4science
Files F103068455

hdfs.proto
No OneTemporary
Actions

File Metadata

Created
Thu, Feb 27, 00:47

hdfs.proto
View Options

/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* These .proto interfaces are private and stable.
* Please see http://wiki.apache.org/hadoop/Compatibility
* for what changes are allowed for a *stable* .proto interface.
*/
// This file contains protocol buffers that are used throughout HDFS -- i.e.
// by the client, server, and data transfer protocols.
option java_package = "org.apache.hadoop.hdfs.protocol.proto";
option java_outer_classname = "HdfsProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdfs;
import "Security.proto";
/**
* Extended block idenfies a block
*/
message ExtendedBlockProto {
required string poolId = 1; // Block pool id - gloablly unique across clusters
required uint64 blockId = 2; // the local id within a pool
required uint64 generationStamp = 3;
optional uint64 numBytes = 4 [default = 0]; // len does not belong in ebid
// here for historical reasons
}
/**
* Identifies a Datanode
*/
message DatanodeIDProto {
required string ipAddr = 1; // IP address
required string hostName = 2; // hostname
required string datanodeUuid = 3; // UUID assigned to the Datanode. For
// upgraded clusters this is the same
// as the original StorageID of the
// Datanode.
required uint32 xferPort = 4; // data streaming port
required uint32 infoPort = 5; // datanode http port
required uint32 ipcPort = 6; // ipc server port
optional uint32 infoSecurePort = 7 [default = 0]; // datanode https port
}
/**
* DatanodeInfo array
*/
message DatanodeInfosProto {
repeated DatanodeInfoProto datanodes = 1;
}
/**
* The status of a Datanode
*/
message DatanodeInfoProto {
required DatanodeIDProto id = 1;
optional uint64 capacity = 2 [default = 0];
optional uint64 dfsUsed = 3 [default = 0];
optional uint64 remaining = 4 [default = 0];
optional uint64 blockPoolUsed = 5 [default = 0];
optional uint64 lastUpdate = 6 [default = 0];
optional uint32 xceiverCount = 7 [default = 0];
optional string location = 8;
enum AdminState {
NORMAL = 0;
DECOMMISSION_INPROGRESS = 1;
DECOMMISSIONED = 2;
}
optional AdminState adminState = 10 [default = NORMAL];
optional uint64 cacheCapacity = 11 [default = 0];
optional uint64 cacheUsed = 12 [default = 0];
}
/**
* Summary of a file or directory
*/
message ContentSummaryProto {
required uint64 length = 1;
required uint64 fileCount = 2;
required uint64 directoryCount = 3;
required uint64 quota = 4;
required uint64 spaceConsumed = 5;
required uint64 spaceQuota = 6;
}
/**
* Contains a list of paths corresponding to corrupt files and a cookie
* used for iterative calls to NameNode.listCorruptFileBlocks.
*
*/
message CorruptFileBlocksProto {
repeated string files = 1;
required string cookie = 2;
}
/**
* File or Directory permision - same spec as posix
*/
message FsPermissionProto {
required uint32 perm = 1; // Actually a short - only 16bits used
}
/**
* Types of recognized storage media.
*/
enum StorageTypeProto {
DISK = 1;
SSD = 2;
}
/**
* A list of storage IDs.
*/
message StorageUuidsProto {
repeated string storageUuids = 1;
}
/**
* A LocatedBlock gives information about a block and its location.
*/
message LocatedBlockProto {
required ExtendedBlockProto b = 1;
required uint64 offset = 2; // offset of first byte of block in the file
repeated DatanodeInfoProto locs = 3; // Locations ordered by proximity to client ip
required bool corrupt = 4; // true if all replicas of a block are corrupt, else false
// If block has few corrupt replicas, they are filtered and
// their locations are not part of this object
required hadoop.common.TokenProto blockToken = 5;
repeated bool isCached = 6 [packed=true]; // if a location in locs is cached
repeated StorageTypeProto storageTypes = 7;
repeated string storageIDs = 8;
}
message DataEncryptionKeyProto {
required uint32 keyId = 1;
required string blockPoolId = 2;
required bytes nonce = 3;
required bytes encryptionKey = 4;
required uint64 expiryDate = 5;
optional string encryptionAlgorithm = 6;
}
/**
* A set of file blocks and their locations.
*/
message LocatedBlocksProto {
required uint64 fileLength = 1;
repeated LocatedBlockProto blocks = 2;
required bool underConstruction = 3;
optional LocatedBlockProto lastBlock = 4;
required bool isLastBlockComplete = 5;
}
/**
* Status of a file, directory or symlink
* Optionally includes a file's block locations if requested by client on the rpc call.
*/
message HdfsFileStatusProto {
enum FileType {
IS_DIR = 1;
IS_FILE = 2;
IS_SYMLINK = 3;
}
required FileType fileType = 1;
required bytes path = 2; // local name of inode encoded java UTF8
required uint64 length = 3;
required FsPermissionProto permission = 4;
required string owner = 5;
required string group = 6;
required uint64 modification_time = 7;
required uint64 access_time = 8;
// Optional fields for symlink
optional bytes symlink = 9; // if symlink, target encoded java UTF8
// Optional fields for file
optional uint32 block_replication = 10 [default = 0]; // only 16bits used
optional uint64 blocksize = 11 [default = 0];
optional LocatedBlocksProto locations = 12; // suppled only if asked by client
// Optional field for fileId
optional uint64 fileId = 13 [default = 0]; // default as an invalid id
optional int32 childrenNum = 14 [default = -1];
}
/**
* Checksum algorithms/types used in HDFS
* Make sure this enum's integer values match enum values' id properties defined
* in org.apache.hadoop.util.DataChecksum.Type
*/
enum ChecksumTypeProto {
CHECKSUM_NULL = 0;
CHECKSUM_CRC32 = 1;
CHECKSUM_CRC32C = 2;
}
/**
* HDFS Server Defaults
*/
message FsServerDefaultsProto {
required uint64 blockSize = 1;
required uint32 bytesPerChecksum = 2;
required uint32 writePacketSize = 3;
required uint32 replication = 4; // Actually a short - only 16 bits used
required uint32 fileBufferSize = 5;
optional bool encryptDataTransfer = 6 [default = false];
optional uint64 trashInterval = 7 [default = 0];
optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32];
}
/**
* Directory listing
*/
message DirectoryListingProto {
repeated HdfsFileStatusProto partialListing = 1;
required uint32 remainingEntries = 2;
}
/**
* Status of a snapshottable directory: besides the normal information for
* a directory status, also include snapshot quota, number of snapshots, and
* the full path of the parent directory.
*/
message SnapshottableDirectoryStatusProto {
required HdfsFileStatusProto dirStatus = 1;
// Fields specific for snapshottable directory
required uint32 snapshot_quota = 2;
required uint32 snapshot_number = 3;
required bytes parent_fullpath = 4;
}
/**
* Snapshottable directory listing
*/
message SnapshottableDirectoryListingProto {
repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1;
}
/**
* Snapshot diff report entry
*/
message SnapshotDiffReportEntryProto {
required bytes fullpath = 1;
required string modificationLabel = 2;
}
/**
* Snapshot diff report
*/
message SnapshotDiffReportProto {
// full path of the directory where snapshots were taken
required string snapshotRoot = 1;
required string fromSnapshot = 2;
required string toSnapshot = 3;
repeated SnapshotDiffReportEntryProto diffReportEntries = 4;
}
/**
* Common node information shared by all the nodes in the cluster
*/
message StorageInfoProto {
required uint32 layoutVersion = 1; // Layout version of the file system
required uint32 namespceID = 2; // File system namespace ID
required string clusterID = 3; // ID of the cluster
required uint64 cTime = 4; // File system creation time
}
/**
* Information sent by a namenode to identify itself to the primary namenode.
*/
message NamenodeRegistrationProto {
required string rpcAddress = 1; // host:port of the namenode RPC address
required string httpAddress = 2; // host:port of the namenode http server
enum NamenodeRoleProto {
NAMENODE = 1;
BACKUP = 2;
CHECKPOINT = 3;
}
required StorageInfoProto storageInfo = 3; // Node information
optional NamenodeRoleProto role = 4 [default = NAMENODE]; // Namenode role
}
/**
* Unique signature to identify checkpoint transactions.
*/
message CheckpointSignatureProto {
required string blockPoolId = 1;
required uint64 mostRecentCheckpointTxId = 2;
required uint64 curSegmentTxId = 3;
required StorageInfoProto storageInfo = 4;
}
/**
* Command sent from one namenode to another namenode.
*/
message NamenodeCommandProto {
enum Type {
NamenodeCommand = 0; // Base command
CheckPointCommand = 1; // Check point command
}
required uint32 action = 1;
required Type type = 2;
optional CheckpointCommandProto checkpointCmd = 3;
}
/**
* Command returned from primary to checkpointing namenode.
* This command has checkpoint signature that identifies
* checkpoint transaction and is needed for further
* communication related to checkpointing.
*/
message CheckpointCommandProto {
// Unique signature to identify checkpoint transation
required CheckpointSignatureProto signature = 1;
// If true, return transfer image to primary upon the completion of checkpoint
required bool needToReturnImage = 2;
}
/**
* Block information
*/
message BlockProto {
required uint64 blockId = 1;
required uint64 genStamp = 2;
optional uint64 numBytes = 3 [default = 0];
}
/**
* Block and datanodes where is it located
*/
message BlockWithLocationsProto {
required BlockProto block = 1; // Block
repeated string datanodeUuids = 2; // Datanodes with replicas of the block
repeated string storageUuids = 3; // Storages with replicas of the block
}
/**
* List of block with locations
*/
message BlocksWithLocationsProto {
repeated BlockWithLocationsProto blocks = 1;
}
/**
* Editlog information with available transactions
*/
message RemoteEditLogProto {
required uint64 startTxId = 1; // Starting available edit log transaction
required uint64 endTxId = 2; // Ending available edit log transaction
optional bool isInProgress = 3 [default = false];
}
/**
* Enumeration of editlogs available on a remote namenode
*/
message RemoteEditLogManifestProto {
repeated RemoteEditLogProto logs = 1;
}
/**
* Namespace information that describes namespace on a namenode
*/
message NamespaceInfoProto {
required string buildVersion = 1; // Software revision version (e.g. an svn or git revision)
required uint32 unused = 2; // Retained for backward compatibility
required string blockPoolID = 3; // block pool used by the namespace
required StorageInfoProto storageInfo = 4;// Node information
required string softwareVersion = 5; // Software version number (e.g. 2.0.0)
}
/**
* Block access token information
*/
message BlockKeyProto {
required uint32 keyId = 1; // Key identifier
required uint64 expiryDate = 2; // Expiry time in milliseconds
optional bytes keyBytes = 3; // Key secret
}
/**
* Current key and set of block keys at the namenode.
*/
message ExportedBlockKeysProto {
required bool isBlockTokenEnabled = 1;
required uint64 keyUpdateInterval = 2;
required uint64 tokenLifeTime = 3;
required BlockKeyProto currentKey = 4;
repeated BlockKeyProto allKeys = 5;
}
/**
* State of a block replica at a datanode
*/
enum ReplicaStateProto {
FINALIZED = 0; // State of a replica when it is not modified
RBW = 1; // State of replica that is being written to
RWR = 2; // State of replica that is waiting to be recovered
RUR = 3; // State of replica that is under recovery
TEMPORARY = 4; // State of replica that is created for replication
}
/**
* Block that needs to be recovered with at a given location
*/
message RecoveringBlockProto {
required uint64 newGenStamp = 1; // New genstamp post recovery
required LocatedBlockProto block = 2; // Block to be recovered
}
/**
* void request
*/
message VersionRequestProto {
}
/**
* Version response from namenode.
*/
message VersionResponseProto {
required NamespaceInfoProto info = 1;
}
/**
* Information related to a snapshot
* TODO: add more information
*/
message SnapshotInfoProto {
required string snapshotName = 1;
required string snapshotRoot = 2;
required FsPermissionProto permission = 3;
required string owner = 4;
required string group = 5;
required string createTime = 6;
// TODO: do we need access time?
}

Event Timeline

c4science · Help