allRanges;
- try {
- allRanges =
- blob.downloadPageRanges(new BlobRequestOptions(), opContext);
- } catch (StorageException e) {
- throw new IOException(e);
- }
- if (allRanges.size() > 0) {
- if (allRanges.get(0).getStartOffset() != 0) {
- throw badStartRangeException(blob, allRanges.get(0));
- }
- if (allRanges.size() > 1) {
- LOG.warn(String.format(
- "Blob %s has %d page ranges beyond the first range. "
- + "Only reading the first range.",
- blob.getUri(), allRanges.size() - 1));
- }
- numberOfPagesRemaining =
- (allRanges.get(0).getEndOffset() + 1) / PAGE_SIZE;
- } else {
- numberOfPagesRemaining = 0;
- }
- }
-
- /** Return the size of the remaining available bytes
- * if the size is less than or equal to {@link Integer#MAX_VALUE},
- * otherwise, return {@link Integer#MAX_VALUE}.
- *
- * This is to match the behavior of DFSInputStream.available(),
- * which some clients may rely on (HBase write-ahead log reading in
- * particular).
- */
- @Override
- public synchronized int available() throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
- if (pageBlobSize == -1) {
- try {
- pageBlobSize = getPageBlobDataSize(blob, opContext);
- } catch (StorageException e) {
- throw new IOException("Unable to get page blob size.", e);
- }
- }
-
- final long remaining = pageBlobSize - filePosition;
- return remaining <= Integer.MAX_VALUE ?
- (int) remaining : Integer.MAX_VALUE;
- }
-
- @Override
- public synchronized void close() throws IOException {
- closed = true;
- }
-
- private boolean dataAvailableInBuffer() {
- return currentBuffer != null
- && currentBufferOffset < currentBufferLength;
- }
-
- /**
- * Check our buffer and download more from the server if needed.
- * If data is not available in the buffer, method downloads maximum
- * page blob download size (4MB) or if there is less then 4MB left,
- * all remaining pages.
- * If we are on the last page, method will return true even if
- * we reached the end of stream.
- * @return true if there's more data in the buffer, false if buffer is empty
- * and we reached the end of the blob.
- * @throws IOException
- */
- private synchronized boolean ensureDataInBuffer() throws IOException {
- if (dataAvailableInBuffer()) {
- // We still have some data in our buffer.
- return true;
- }
- currentBuffer = null;
- currentBufferOffset = 0;
- currentBufferLength = 0;
- if (numberOfPagesRemaining == 0) {
- // No more data to read.
- return false;
- }
- final long pagesToRead = Math.min(MAX_PAGES_PER_DOWNLOAD,
- numberOfPagesRemaining);
- final int bufferSize = (int) (pagesToRead * PAGE_SIZE);
-
- // Download page to current buffer.
- try {
- // Create a byte array output stream to capture the results of the
- // download.
- ByteArrayOutputStream baos = new ByteArrayOutputStream(bufferSize);
- blob.downloadRange(currentOffsetInBlob, bufferSize, baos,
- withMD5Checking(), opContext);
- validateDataIntegrity(baos.toByteArray());
- } catch (StorageException e) {
- throw new IOException(e);
- }
- numberOfPagesRemaining -= pagesToRead;
- currentOffsetInBlob += bufferSize;
-
- return true;
- }
-
- private void validateDataIntegrity(byte[] buffer)
- throws IOException {
-
- if (buffer.length % PAGE_SIZE != 0) {
- throw new AssertionError("Unexpected buffer size: "
- + buffer.length);
- }
-
- int bufferLength = 0;
- int numberOfPages = buffer.length / PAGE_SIZE;
- long totalPagesAfterCurrent = numberOfPagesRemaining;
-
- for (int page = 0; page < numberOfPages; page++) {
- // Calculate the number of pages that exist in the blob after this one
- totalPagesAfterCurrent--;
-
- short currentPageSize = getPageSize(blob, buffer, page * PAGE_SIZE);
-
- // Only the last page can be partially filled.
- if (currentPageSize < PAGE_DATA_SIZE
- && totalPagesAfterCurrent > 0) {
- throw fileCorruptException(blob, String.format(
- "Page with partial data found in the middle (%d pages from the"
- + " end) that only has %d bytes of data.",
- totalPagesAfterCurrent, currentPageSize));
- }
- bufferLength += currentPageSize + PAGE_HEADER_SIZE;
- }
-
- currentBufferOffset = PAGE_HEADER_SIZE;
- currentBufferLength = bufferLength;
- currentBuffer = buffer;
- }
-
- // Reads the page size from the page header at the given offset.
- private static short getPageSize(CloudPageBlobWrapper blob,
- byte[] data, int offset) throws IOException {
- short pageSize = toShort(data[offset], data[offset + 1]);
- if (pageSize < 0 || pageSize > PAGE_DATA_SIZE) {
- throw fileCorruptException(blob, String.format(
- "Unexpected page size in the header: %d.",
- pageSize));
- }
- return pageSize;
- }
-
- @Override
- public synchronized int read(byte[] outputBuffer, int offset, int len)
- throws IOException {
- // If len is zero return 0 per the InputStream contract
- if (len == 0) {
- return 0;
- }
-
- int numberOfBytesRead = 0;
- while (len > 0) {
- if (!ensureDataInBuffer()) {
- break;
- }
- int bytesRemainingInCurrentPage = getBytesRemainingInCurrentPage();
- int numBytesToRead = Math.min(len, bytesRemainingInCurrentPage);
- System.arraycopy(currentBuffer, currentBufferOffset, outputBuffer,
- offset, numBytesToRead);
- numberOfBytesRead += numBytesToRead;
- offset += numBytesToRead;
- len -= numBytesToRead;
- if (numBytesToRead == bytesRemainingInCurrentPage) {
- // We've finished this page, move on to the next.
- advancePagesInBuffer(1);
- } else {
- currentBufferOffset += numBytesToRead;
- }
- }
-
- // if outputBuffer len is > 0 and zero bytes were read, we reached
- // an EOF
- if (numberOfBytesRead == 0) {
- return -1;
- }
-
- filePosition += numberOfBytesRead;
- return numberOfBytesRead;
- }
-
- @Override
- public int read() throws IOException {
- byte[] oneByte = new byte[1];
- int result = read(oneByte);
- if (result < 0) {
- return result;
- }
- return oneByte[0];
- }
-
- /**
- * Skips over and discards n bytes of data from this input
- * stream. The skip method may, for a variety of reasons, end
- * up skipping over some smaller number of bytes, possibly 0.
- * This may result from any of a number of conditions; reaching end of file
- * before n bytes have been skipped is only one possibility.
- * The actual number of bytes skipped is returned. If {@code n} is
- * negative, the {@code skip} method for class {@code InputStream} always
- * returns 0, and no bytes are skipped. Subclasses may handle the negative
- * value differently.
- *
- * The skip method of this class creates a
- * byte array and then repeatedly reads into it until n bytes
- * have been read or the end of the stream has been reached. Subclasses are
- * encouraged to provide a more efficient implementation of this method.
- * For instance, the implementation may depend on the ability to seek.
- *
- * @param n the number of bytes to be skipped.
- * @return the actual number of bytes skipped.
- * @exception IOException if the stream does not support seek,
- * or if some other I/O error occurs.
- */
- @Override
- public synchronized long skip(long n) throws IOException {
- long skipped = skipImpl(n);
- filePosition += skipped; // track the position in the stream
- return skipped;
- }
-
- private long skipImpl(long n) throws IOException {
-
- if (n == 0) {
- return 0;
- }
-
- // First skip within the current buffer as much as possible.
- long skippedWithinBuffer = skipWithinBuffer(n);
- if (skippedWithinBuffer > n) {
- // TO CONSIDER: Using a contracts framework such as Google's cofoja for
- // these post-conditions.
- throw new AssertionError(String.format(
- "Bug in skipWithinBuffer: it skipped over %d bytes when asked to "
- + "skip %d bytes.", skippedWithinBuffer, n));
- }
- n -= skippedWithinBuffer;
- long skipped = skippedWithinBuffer;
-
- if (n == 0) {
- return skipped;
- }
-
- if (numberOfPagesRemaining == 0) {
- throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
- } else if (numberOfPagesRemaining > 1) {
- // skip over as many pages as we can, but we must read the last
- // page as it may not be full
- long pagesToSkipOver = Math.min(n / PAGE_DATA_SIZE,
- numberOfPagesRemaining - 1);
- numberOfPagesRemaining -= pagesToSkipOver;
- currentOffsetInBlob += pagesToSkipOver * PAGE_SIZE;
- skipped += pagesToSkipOver * PAGE_DATA_SIZE;
- n -= pagesToSkipOver * PAGE_DATA_SIZE;
- }
-
- if (n == 0) {
- return skipped;
- }
-
- // Now read in at the current position, and skip within current buffer.
- if (!ensureDataInBuffer()) {
- return skipped;
- }
- return skipped + skipWithinBuffer(n);
- }
-
- /**
- * Skip over n bytes within the current buffer or just over skip the whole
- * buffer if n is greater than the bytes remaining in the buffer.
- * @param n The number of data bytes to skip.
- * @return The number of bytes actually skipped.
- * @throws IOException if data corruption found in the buffer.
- */
- private long skipWithinBuffer(long n) throws IOException {
- if (!dataAvailableInBuffer()) {
- return 0;
- }
- long skipped = 0;
- // First skip within the current page.
- skipped = skipWithinCurrentPage(n);
- if (skipped > n) {
- throw new AssertionError(String.format(
- "Bug in skipWithinCurrentPage: it skipped over %d bytes when asked"
- + " to skip %d bytes.", skipped, n));
- }
- n -= skipped;
- if (n == 0 || !dataAvailableInBuffer()) {
- return skipped;
- }
-
- // Calculate how many whole pages (pages before the possibly partially
- // filled last page) remain.
- int currentPageIndex = currentBufferOffset / PAGE_SIZE;
- int numberOfPagesInBuffer = currentBuffer.length / PAGE_SIZE;
- int wholePagesRemaining = numberOfPagesInBuffer - currentPageIndex - 1;
-
- if (n < (PAGE_DATA_SIZE * wholePagesRemaining)) {
- // I'm within one of the whole pages remaining, skip in there.
- advancePagesInBuffer((int) (n / PAGE_DATA_SIZE));
- currentBufferOffset += n % PAGE_DATA_SIZE;
- return n + skipped;
- }
-
- // Skip over the whole pages.
- advancePagesInBuffer(wholePagesRemaining);
- skipped += wholePagesRemaining * PAGE_DATA_SIZE;
- n -= wholePagesRemaining * PAGE_DATA_SIZE;
-
- // At this point we know we need to skip to somewhere in the last page,
- // or just go to the end.
- return skipWithinCurrentPage(n) + skipped;
- }
-
- /**
- * Skip over n bytes within the current page or just over skip the whole
- * page if n is greater than the bytes remaining in the page.
- * @param n The number of data bytes to skip.
- * @return The number of bytes actually skipped.
- * @throws IOException if data corruption found in the buffer.
- */
- private long skipWithinCurrentPage(long n) throws IOException {
- int remainingBytesInCurrentPage = getBytesRemainingInCurrentPage();
- if (n <= remainingBytesInCurrentPage) {
- currentBufferOffset += n;
- return n;
- } else {
- advancePagesInBuffer(1);
- return remainingBytesInCurrentPage;
- }
- }
-
- /**
- * Gets the number of bytes remaining within the current page in the buffer.
- * @return The number of bytes remaining.
- * @throws IOException if data corruption found in the buffer.
- */
- private int getBytesRemainingInCurrentPage() throws IOException {
- if (!dataAvailableInBuffer()) {
- return 0;
- }
- // Calculate our current position relative to the start of the current
- // page.
- int currentDataOffsetInPage =
- (currentBufferOffset % PAGE_SIZE) - PAGE_HEADER_SIZE;
- int pageBoundary = getCurrentPageStartInBuffer();
- // Get the data size of the current page from the header.
- short sizeOfCurrentPage = getPageSize(blob, currentBuffer, pageBoundary);
- return sizeOfCurrentPage - currentDataOffsetInPage;
- }
-
- private static IOException badStartRangeException(CloudPageBlobWrapper blob,
- PageRange startRange) {
- return fileCorruptException(blob, String.format(
- "Page blobs for ASV should always use a page range starting at byte 0. "
- + "This starts at byte %d.",
- startRange.getStartOffset()));
- }
-
- private void advancePagesInBuffer(int numberOfPages) {
- currentBufferOffset =
- getCurrentPageStartInBuffer()
- + (numberOfPages * PAGE_SIZE)
- + PAGE_HEADER_SIZE;
- }
-
- private int getCurrentPageStartInBuffer() {
- return PAGE_SIZE * (currentBufferOffset / PAGE_SIZE);
- }
-
- private static IOException fileCorruptException(CloudPageBlobWrapper blob,
- String reason) {
- return new IOException(String.format(
- "The page blob: '%s' is corrupt or has an unexpected format: %s.",
- blob.getUri(), reason));
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
deleted file mode 100644
index f77a6b805140d..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
+++ /dev/null
@@ -1,607 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.PAGE_DATA_SIZE;
-import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.PAGE_HEADER_SIZE;
-import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.PAGE_SIZE;
-import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.fromShort;
-import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.withMD5Checking;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.Locale;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.fs.StreamCapabilities;
-import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.CloudPageBlob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An output stream that write file data to a page blob stored using ASV's
- * custom format.
- */
-final class PageBlobOutputStream extends OutputStream implements Syncable, StreamCapabilities {
- /**
- * The maximum number of raw bytes Azure Storage allows us to upload in a
- * single request (4 MB).
- */
- private static final int MAX_RAW_BYTES_PER_REQUEST = 4 * 1024 * 1024;
- /**
- * The maximum number of pages Azure Storage allows us to upload in a
- * single request.
- */
- private static final int MAX_PAGES_IN_REQUEST =
- MAX_RAW_BYTES_PER_REQUEST / PAGE_SIZE;
- /**
- * The maximum number of data bytes (header not included) we can upload
- * in a single request. I'm limiting it to (N - 1) pages to account for
- * the possibility that we may have to rewrite the previous request's
- * last page.
- */
- private static final int MAX_DATA_BYTES_PER_REQUEST =
- PAGE_DATA_SIZE * (MAX_PAGES_IN_REQUEST - 1);
-
- private final CloudPageBlobWrapper blob;
- private final OperationContext opContext;
-
- /**
- * If the IO thread encounters an error, it'll store it here.
- */
- private volatile IOException lastError;
-
- /**
- * Current size of the page blob in bytes. It may be extended if the file
- * gets full.
- */
- private long currentBlobSize;
- /**
- * The current byte offset we're at in the blob (how many bytes we've
- * uploaded to the server).
- */
- private long currentBlobOffset;
- /**
- * The data in the last page that we wrote to the server, in case we have to
- * overwrite it in the new request.
- */
- private byte[] previousLastPageDataWritten = new byte[0];
- /**
- * The current buffer we're writing to before sending to the server.
- */
- private ByteArrayOutputStream outBuffer;
- /**
- * The task queue for writing to the server.
- */
- private final LinkedBlockingQueue ioQueue;
- /**
- * The thread pool we're using for writing to the server. Note that the IO
- * write is NOT designed for parallelism, so there can only be one thread
- * in that pool (I'm using the thread pool mainly for the lifetime management
- * capabilities, otherwise I'd have just used a simple Thread).
- */
- private final ThreadPoolExecutor ioThreadPool;
-
- // The last task given to the ioThreadPool to execute, to allow
- // waiting until it's done.
- private WriteRequest lastQueuedTask;
- // Whether the stream has been closed.
- private boolean closed = false;
-
- public static final Logger LOG = LoggerFactory.getLogger(AzureNativeFileSystemStore.class);
-
- // Set the minimum page blob file size to 128MB, which is >> the default
- // block size of 32MB. This default block size is often used as the
- // hbase.regionserver.hlog.blocksize.
- // The goal is to have a safe minimum size for HBase log files to allow them
- // to be filled and rolled without exceeding the minimum size. A larger size
- // can be used by setting the fs.azure.page.blob.size configuration variable.
- public static final long PAGE_BLOB_MIN_SIZE = 128L * 1024L * 1024L;
-
- // The default and minimum amount to extend a page blob by if it starts
- // to get full.
- public static final long
- PAGE_BLOB_DEFAULT_EXTENSION_SIZE = 128L * 1024L * 1024L;
-
- // The configured page blob extension size (either the default, or if greater,
- // the value configured in fs.azure.page.blob.extension.size
- private long configuredPageBlobExtensionSize;
-
- /**
- * Constructs an output stream over the given page blob.
- *
- * @param blob the blob that this stream is associated with.
- * @param opContext an object used to track the execution of the operation
- * @throws StorageException if anything goes wrong creating the blob.
- */
- public PageBlobOutputStream(final CloudPageBlobWrapper blob,
- final OperationContext opContext,
- final Configuration conf) throws StorageException {
- this.blob = blob;
- this.outBuffer = new ByteArrayOutputStream();
- this.opContext = opContext;
- this.lastQueuedTask = null;
- this.ioQueue = new LinkedBlockingQueue();
-
- // As explained above: the IO writes are not designed for parallelism,
- // so we only have one thread in this thread pool.
- this.ioThreadPool = new ThreadPoolExecutor(1, 1, 2, TimeUnit.SECONDS,
- ioQueue);
-
-
-
- // Make page blob files have a size that is the greater of a
- // minimum size, or the value of fs.azure.page.blob.size from configuration.
- long pageBlobConfigSize = conf.getLong("fs.azure.page.blob.size", 0);
- LOG.debug("Read value of fs.azure.page.blob.size as " + pageBlobConfigSize
- + " from configuration (0 if not present).");
- long pageBlobSize = Math.max(PAGE_BLOB_MIN_SIZE, pageBlobConfigSize);
-
- // Ensure that the pageBlobSize is a multiple of page size.
- if (pageBlobSize % PAGE_SIZE != 0) {
- pageBlobSize += PAGE_SIZE - pageBlobSize % PAGE_SIZE;
- }
- blob.create(pageBlobSize, new BlobRequestOptions(), opContext);
- currentBlobSize = pageBlobSize;
-
- // Set the page blob extension size. It must be a minimum of the default
- // value.
- configuredPageBlobExtensionSize =
- conf.getLong("fs.azure.page.blob.extension.size", 0);
- if (configuredPageBlobExtensionSize < PAGE_BLOB_DEFAULT_EXTENSION_SIZE) {
- configuredPageBlobExtensionSize = PAGE_BLOB_DEFAULT_EXTENSION_SIZE;
- }
-
- // make sure it is a multiple of the page size
- if (configuredPageBlobExtensionSize % PAGE_SIZE != 0) {
- configuredPageBlobExtensionSize +=
- PAGE_SIZE - configuredPageBlobExtensionSize % PAGE_SIZE;
- }
- }
-
- private void checkStreamState() throws IOException {
- if (lastError != null) {
- throw lastError;
- }
- }
-
- /**
- * Query the stream for a specific capability.
- *
- * @param capability string to query the stream support for.
- * @return true for hsync and hflush.
- */
- @Override
- public boolean hasCapability(String capability) {
- switch (capability.toLowerCase(Locale.ENGLISH)) {
- case StreamCapabilities.HSYNC:
- case StreamCapabilities.HFLUSH:
- return true;
- default:
- return false;
- }
- }
-
- /**
- * Closes this output stream and releases any system resources associated with
- * this stream. If any data remains in the buffer it is committed to the
- * service.
- */
- @Override
- public synchronized void close() throws IOException {
- if (closed) {
- return;
- }
-
- LOG.debug("Closing page blob output stream.");
- flush();
- checkStreamState();
- ioThreadPool.shutdown();
- try {
- LOG.debug(ioThreadPool.toString());
- if (!ioThreadPool.awaitTermination(10, TimeUnit.MINUTES)) {
- LOG.debug("Timed out after 10 minutes waiting for IO requests to finish");
- NativeAzureFileSystemHelper.logAllLiveStackTraces();
- LOG.debug(ioThreadPool.toString());
- throw new IOException("Timed out waiting for IO requests to finish");
- }
- } catch (InterruptedException e) {
- LOG.debug("Caught InterruptedException");
-
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- }
-
- closed = true;
- }
-
-
-
- /**
- * A single write request for data to write to Azure storage.
- */
- private class WriteRequest implements Runnable {
- private final byte[] dataPayload;
- private final CountDownLatch doneSignal = new CountDownLatch(1);
-
- public WriteRequest(byte[] dataPayload) {
- this.dataPayload = dataPayload;
- }
-
- public void waitTillDone() throws InterruptedException {
- doneSignal.await();
- }
-
- @Override
- public void run() {
- try {
- LOG.debug("before runInternal()");
- runInternal();
- LOG.debug("after runInternal()");
- } finally {
- doneSignal.countDown();
- }
- }
-
- private void runInternal() {
- if (lastError != null) {
- // We're already in an error state, no point doing anything.
- return;
- }
- if (dataPayload.length == 0) {
- // Nothing to do.
- return;
- }
-
- // Since we have to rewrite the last request's last page's data
- // (may be empty), total data size is our data plus whatever was
- // left from there.
- final int totalDataBytes = dataPayload.length
- + previousLastPageDataWritten.length;
- // Calculate the total number of pages we're writing to the server.
- final int numberOfPages = (totalDataBytes / PAGE_DATA_SIZE)
- + (totalDataBytes % PAGE_DATA_SIZE == 0 ? 0 : 1);
- // Fill up the raw bytes we're writing.
- byte[] rawPayload = new byte[numberOfPages * PAGE_SIZE];
- // Keep track of the size of the last page we uploaded.
- int currentLastPageDataSize = -1;
- for (int page = 0; page < numberOfPages; page++) {
- // Our current byte offset in the data.
- int dataOffset = page * PAGE_DATA_SIZE;
- // Our current byte offset in the raw buffer.
- int rawOffset = page * PAGE_SIZE;
- // The size of the data in the current page.
- final short currentPageDataSize = (short) Math.min(PAGE_DATA_SIZE,
- totalDataBytes - dataOffset);
- // Save off this page's size as the potential last page's size.
- currentLastPageDataSize = currentPageDataSize;
-
- // Write out the page size in the header.
- final byte[] header = fromShort(currentPageDataSize);
- System.arraycopy(header, 0, rawPayload, rawOffset, header.length);
- rawOffset += header.length;
-
- int bytesToCopyFromDataPayload = currentPageDataSize;
- if (dataOffset < previousLastPageDataWritten.length) {
- // First write out the last page's data.
- final int bytesToCopyFromLastPage = Math.min(currentPageDataSize,
- previousLastPageDataWritten.length - dataOffset);
- System.arraycopy(previousLastPageDataWritten, dataOffset,
- rawPayload, rawOffset, bytesToCopyFromLastPage);
- bytesToCopyFromDataPayload -= bytesToCopyFromLastPage;
- rawOffset += bytesToCopyFromLastPage;
- dataOffset += bytesToCopyFromLastPage;
- }
-
- if (dataOffset >= previousLastPageDataWritten.length) {
- // Then write the current payload's data.
- System.arraycopy(dataPayload,
- dataOffset - previousLastPageDataWritten.length,
- rawPayload, rawOffset, bytesToCopyFromDataPayload);
- }
- }
-
- // Raw payload constructed, ship it off to the server.
- writePayloadToServer(rawPayload);
-
- // Post-send bookkeeping.
- currentBlobOffset += rawPayload.length;
- if (currentLastPageDataSize < PAGE_DATA_SIZE) {
- // Partial page, save it off so it's overwritten in the next request.
- final int startOffset = (numberOfPages - 1) * PAGE_SIZE + PAGE_HEADER_SIZE;
- previousLastPageDataWritten = Arrays.copyOfRange(rawPayload,
- startOffset,
- startOffset + currentLastPageDataSize);
- // Since we're rewriting this page, set our current offset in the server
- // to that page's beginning.
- currentBlobOffset -= PAGE_SIZE;
- } else {
- // It wasn't a partial page, we won't need to rewrite it.
- previousLastPageDataWritten = new byte[0];
- }
-
- // Extend the file if we need more room in the file. This typically takes
- // less than 200 milliseconds if it has to actually be done,
- // so it is okay to include it in a write and won't cause a long pause.
- // Other writes can be queued behind this write in any case.
- conditionalExtendFile();
- }
-
- /**
- * Writes the given raw payload to Azure Storage at the current blob
- * offset.
- */
- private void writePayloadToServer(byte[] rawPayload) {
- final ByteArrayInputStream wrapperStream =
- new ByteArrayInputStream(rawPayload);
- LOG.debug("writing payload of " + rawPayload.length + " bytes to Azure page blob");
- try {
- long start = System.currentTimeMillis();
- blob.uploadPages(wrapperStream, currentBlobOffset, rawPayload.length,
- withMD5Checking(), PageBlobOutputStream.this.opContext);
- long end = System.currentTimeMillis();
- LOG.trace("Azure uploadPages time for " + rawPayload.length + " bytes = " + (end - start));
- } catch (IOException ex) {
- LOG.debug(ExceptionUtils.getStackTrace(ex));
- lastError = ex;
- } catch (StorageException ex) {
- LOG.debug(ExceptionUtils.getStackTrace(ex));
- lastError = new IOException(ex);
- }
- if (lastError != null) {
- LOG.debug("Caught error in PageBlobOutputStream#writePayloadToServer()");
- }
- }
- }
-
- private synchronized void flushIOBuffers() {
- if (outBuffer.size() == 0) {
- return;
- }
- lastQueuedTask = new WriteRequest(outBuffer.toByteArray());
- ioThreadPool.execute(lastQueuedTask);
- outBuffer = new ByteArrayOutputStream();
- }
-
- @VisibleForTesting
- synchronized void waitForLastFlushCompletion() throws IOException {
- try {
- if (lastQueuedTask != null) {
- lastQueuedTask.waitTillDone();
- }
- } catch (InterruptedException e1) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Extend the page blob file if we are close to the end.
- */
- private void conditionalExtendFile() {
-
- // maximum allowed size of an Azure page blob (1 terabyte)
- final long MAX_PAGE_BLOB_SIZE = 1024L * 1024L * 1024L * 1024L;
-
- // If blob is already at the maximum size, then don't try to extend it.
- if (currentBlobSize == MAX_PAGE_BLOB_SIZE) {
- return;
- }
-
- // If we are within the maximum write size of the end of the file,
- if (currentBlobSize - currentBlobOffset <= MAX_RAW_BYTES_PER_REQUEST) {
-
- // Extend the file. Retry up to 3 times with back-off.
- CloudPageBlob cloudPageBlob = (CloudPageBlob) blob.getBlob();
- long newSize = currentBlobSize + configuredPageBlobExtensionSize;
-
- // Make sure we don't exceed maximum blob size.
- if (newSize > MAX_PAGE_BLOB_SIZE) {
- newSize = MAX_PAGE_BLOB_SIZE;
- }
- final int MAX_RETRIES = 3;
- int retries = 1;
- boolean resizeDone = false;
- while(!resizeDone && retries <= MAX_RETRIES) {
- try {
- cloudPageBlob.resize(newSize);
- resizeDone = true;
- currentBlobSize = newSize;
- } catch (StorageException e) {
- LOG.warn("Failed to extend size of " + cloudPageBlob.getUri());
- try {
-
- // sleep 2, 8, 18 seconds for up to 3 retries
- Thread.sleep(2000 * retries * retries);
- } catch (InterruptedException e1) {
-
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- }
- } finally {
- retries++;
- }
- }
- }
- }
-
- /**
- * Flushes this output stream and forces any buffered output bytes to be
- * written out. If any data remains in the buffer it is committed to the
- * service. Data is queued for writing but not forced out to the service
- * before the call returns.
- */
- @Override
- public void flush() throws IOException {
- checkStreamState();
- flushIOBuffers();
- }
-
- /**
- * Writes b.length bytes from the specified byte array to this output stream.
- *
- * @param data
- * the byte array to write.
- *
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- @Override
- public void write(final byte[] data) throws IOException {
- write(data, 0, data.length);
- }
-
- /**
- * Writes length bytes from the specified byte array starting at offset to
- * this output stream.
- *
- * @param data
- * the byte array to write.
- * @param offset
- * the start offset in the data.
- * @param length
- * the number of bytes to write.
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- @Override
- public void write(final byte[] data, final int offset, final int length)
- throws IOException {
- if (offset < 0 || length < 0 || length > data.length - offset) {
- throw new IndexOutOfBoundsException();
- }
-
- writeInternal(data, offset, length);
- }
-
- /**
- * Writes the specified byte to this output stream. The general contract for
- * write is that one byte is written to the output stream. The byte to be
- * written is the eight low-order bits of the argument b. The 24 high-order
- * bits of b are ignored.
- *
- * @param byteVal
- * the byteValue to write.
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- @Override
- public void write(final int byteVal) throws IOException {
- write(new byte[] { (byte) (byteVal & 0xFF) });
- }
-
- /**
- * Writes the data to the buffer and triggers writes to the service as needed.
- *
- * @param data
- * the byte array to write.
- * @param offset
- * the start offset in the data.
- * @param length
- * the number of bytes to write.
- * @throws IOException
- * if an I/O error occurs. In particular, an IOException may be
- * thrown if the output stream has been closed.
- */
- private synchronized void writeInternal(final byte[] data, int offset,
- int length) throws IOException {
- while (length > 0) {
- checkStreamState();
- final int availableBufferBytes = MAX_DATA_BYTES_PER_REQUEST
- - this.outBuffer.size();
- final int nextWrite = Math.min(availableBufferBytes, length);
-
- outBuffer.write(data, offset, nextWrite);
- offset += nextWrite;
- length -= nextWrite;
-
- if (outBuffer.size() > MAX_DATA_BYTES_PER_REQUEST) {
- throw new RuntimeException("Internal error: maximum write size " +
- Integer.toString(MAX_DATA_BYTES_PER_REQUEST) + "exceeded.");
- }
-
- if (outBuffer.size() == MAX_DATA_BYTES_PER_REQUEST) {
- flushIOBuffers();
- }
- }
- }
-
- /**
- * Force all data in the output stream to be written to Azure storage.
- * Wait to return until this is complete.
- */
- @Override
- public synchronized void hsync() throws IOException {
- LOG.debug("Entering PageBlobOutputStream#hsync().");
- long start = System.currentTimeMillis();
- flush();
- LOG.debug(ioThreadPool.toString());
- try {
- if (lastQueuedTask != null) {
- lastQueuedTask.waitTillDone();
- }
- } catch (InterruptedException e1) {
-
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- }
- checkStreamState();
- LOG.debug("Leaving PageBlobOutputStream#hsync(). Total hsync duration = "
- + (System.currentTimeMillis() - start) + " msec.");
- }
-
- @Override
- public void hflush() throws IOException {
-
- // hflush is required to force data to storage, so call hsync,
- // which does that.
- hsync();
- }
-
- @Deprecated
- public void sync() throws IOException {
-
- // Sync has been deprecated in favor of hflush.
- hflush();
- }
-
- // For unit testing purposes: kill the IO threads.
- @VisibleForTesting
- void killIoThreads() {
- ioThreadPool.shutdownNow();
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
deleted file mode 100644
index 473fa54f97c83..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import org.apache.http.NameValuePair;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URIBuilder;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE;
-
-/**
- * Class implementing a RemoteSASKeyGenerator. This class
- * uses the url passed in via the Configuration to make a
- * rest call to generate the required SAS Key.
- */
-public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl {
-
- public static final Logger LOG =
- LoggerFactory.getLogger(AzureNativeFileSystemStore.class);
- private static final ObjectReader RESPONSE_READER = new ObjectMapper()
- .readerFor(RemoteSASKeyGenerationResponse.class);
-
- /**
- * Configuration parameter name expected in the Configuration
- * object to provide the url of the remote service {@value}
- */
- public static final String KEY_CRED_SERVICE_URLS =
- "fs.azure.cred.service.urls";
- /**
- * Configuration key to enable http retry policy for SAS Key generation. {@value}
- */
- public static final String
- SAS_KEY_GENERATOR_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY =
- "fs.azure.saskeygenerator.http.retry.policy.enabled";
- /**
- * Configuration key for SAS Key Generation http retry policy spec. {@value}
- */
- public static final String
- SAS_KEY_GENERATOR_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY =
- "fs.azure.saskeygenerator.http.retry.policy.spec";
- /**
- * Container SAS Key generation OP name. {@value}
- */
- private static final String CONTAINER_SAS_OP = "GET_CONTAINER_SAS";
- /**
- * Relative Blob SAS Key generation OP name. {@value}
- */
- private static final String BLOB_SAS_OP = "GET_RELATIVE_BLOB_SAS";
- /**
- * Query parameter specifying the expiry period to be used for sas key
- * {@value}
- */
- private static final String SAS_EXPIRY_QUERY_PARAM_NAME = "sas_expiry";
- /**
- * Query parameter name for the storage account. {@value}
- */
- private static final String STORAGE_ACCOUNT_QUERY_PARAM_NAME =
- "storage_account";
- /**
- * Query parameter name for the storage account container. {@value}
- */
- private static final String CONTAINER_QUERY_PARAM_NAME = "container";
- /**
- * Query parameter name for the relative path inside the storage
- * account container. {@value}
- */
- private static final String RELATIVE_PATH_QUERY_PARAM_NAME = "relative_path";
- /**
- * SAS Key Generation Remote http client retry policy spec. {@value}
- */
- private static final String
- SAS_KEY_GENERATOR_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
- "10,3,100,2";
- /**
- * Saskey caching period
- */
- private static final String SASKEY_CACHEENTRY_EXPIRY_PERIOD =
- "fs.azure.saskey.cacheentry.expiry.period";
-
- private WasbRemoteCallHelper remoteCallHelper = null;
- private boolean isKerberosSupportEnabled;
- private boolean isSpnegoTokenCacheEnabled;
- private RetryPolicy retryPolicy;
- private String[] commaSeparatedUrls;
- private CachingAuthorizer cache;
-
- private static final int HOURS_IN_DAY = 24;
- private static final int MINUTES_IN_HOUR = 60;
-
- public RemoteSASKeyGeneratorImpl(Configuration conf) {
- super(conf);
- }
-
- public void initialize(Configuration conf) throws IOException {
-
- LOG.debug("Initializing RemoteSASKeyGeneratorImpl instance");
-
- this.retryPolicy = RetryUtils.getMultipleLinearRandomRetry(conf,
- SAS_KEY_GENERATOR_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY, true,
- SAS_KEY_GENERATOR_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
- SAS_KEY_GENERATOR_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT);
-
- this.isKerberosSupportEnabled =
- conf.getBoolean(Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
- this.isSpnegoTokenCacheEnabled =
- conf.getBoolean(Constants.AZURE_ENABLE_SPNEGO_TOKEN_CACHE, true);
- this.commaSeparatedUrls = conf.getTrimmedStrings(KEY_CRED_SERVICE_URLS);
- if (this.commaSeparatedUrls == null || this.commaSeparatedUrls.length <= 0) {
- throw new IOException(
- KEY_CRED_SERVICE_URLS + " config not set" + " in configuration.");
- }
- if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled()) {
- this.remoteCallHelper = new SecureWasbRemoteCallHelper(retryPolicy, false,
- isSpnegoTokenCacheEnabled);
- } else {
- this.remoteCallHelper = new WasbRemoteCallHelper(retryPolicy);
- }
-
- /* Expire the cache entry five minutes before the actual saskey expiry, so that we never encounter a case
- * where a stale sas-key-entry is picked up from the cache; which is expired on use.
- */
- long sasKeyExpiryPeriodInMinutes = getSasKeyExpiryPeriod() * HOURS_IN_DAY * MINUTES_IN_HOUR; // sas-expiry is in days, convert into mins
- long cacheEntryDurationInMinutes =
- conf.getTimeDuration(SASKEY_CACHEENTRY_EXPIRY_PERIOD, sasKeyExpiryPeriodInMinutes, TimeUnit.MINUTES);
- cacheEntryDurationInMinutes = (cacheEntryDurationInMinutes > (sasKeyExpiryPeriodInMinutes - 5))
- ? (sasKeyExpiryPeriodInMinutes - 5)
- : cacheEntryDurationInMinutes;
- this.cache = new CachingAuthorizer<>(cacheEntryDurationInMinutes, "SASKEY");
- this.cache.init(conf);
- LOG.debug("Initialization of RemoteSASKeyGenerator instance successful");
- }
-
- @Override
- public URI getContainerSASUri(String storageAccount,
- String container) throws SASKeyGenerationException {
- RemoteSASKeyGenerationResponse sasKeyResponse = null;
- try {
- CachedSASKeyEntry cacheKey = new CachedSASKeyEntry(storageAccount, container, "/");
- URI cacheResult = cache.get(cacheKey);
- if (cacheResult != null) {
- return cacheResult;
- }
-
- LOG.debug("Generating Container SAS Key: Storage Account {}, Container {}", storageAccount, container);
- URIBuilder uriBuilder = new URIBuilder();
- uriBuilder.setPath("/" + CONTAINER_SAS_OP);
- uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME, storageAccount);
- uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME, container);
- uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME,
- "" + getSasKeyExpiryPeriod());
-
- sasKeyResponse = makeRemoteRequest(commaSeparatedUrls, uriBuilder.getPath(),
- uriBuilder.getQueryParams());
-
- if (sasKeyResponse.getResponseCode() == REMOTE_CALL_SUCCESS_CODE) {
- URI sasKey = new URI(sasKeyResponse.getSasKey());
- cache.put(cacheKey, sasKey);
- return sasKey;
- } else {
- throw new SASKeyGenerationException(
- "Remote Service encountered error in SAS Key generation : "
- + sasKeyResponse.getResponseMessage());
- }
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException"
- + " while building the HttpGetRequest to remote service for ",
- uriSyntaxEx);
- }
- }
-
- @Override
- public URI getRelativeBlobSASUri(String storageAccount,
- String container, String relativePath) throws SASKeyGenerationException {
-
- try {
- CachedSASKeyEntry cacheKey = new CachedSASKeyEntry(storageAccount, container, relativePath);
- URI cacheResult = cache.get(cacheKey);
- if (cacheResult != null) {
- return cacheResult;
- }
-
- LOG.debug("Generating RelativePath SAS Key for relativePath {} inside Container {} inside Storage Account {}",
- relativePath, container, storageAccount);
-
- URIBuilder uriBuilder = new URIBuilder();
- uriBuilder.setPath("/" + BLOB_SAS_OP);
- uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME, storageAccount);
- uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME, container);
- uriBuilder.addParameter(RELATIVE_PATH_QUERY_PARAM_NAME, relativePath);
- uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME,
- "" + getSasKeyExpiryPeriod());
-
- RemoteSASKeyGenerationResponse sasKeyResponse =
- makeRemoteRequest(commaSeparatedUrls, uriBuilder.getPath(),
- uriBuilder.getQueryParams());
- if (sasKeyResponse.getResponseCode() == REMOTE_CALL_SUCCESS_CODE) {
- URI sasKey = new URI(sasKeyResponse.getSasKey());
- cache.put(cacheKey, sasKey);
- return sasKey;
- } else {
- throw new SASKeyGenerationException(
- "Remote Service encountered error in SAS Key generation : "
- + sasKeyResponse.getResponseMessage());
- }
- } catch (URISyntaxException uriSyntaxEx) {
- throw new SASKeyGenerationException("Encountered URISyntaxException"
- + " while building the HttpGetRequest to " + " remote service",
- uriSyntaxEx);
- }
- }
-
- /**
- * Helper method to make a remote request.
- *
- * @param urls - Urls to use for the remote request
- * @param path - hadoop.auth token for the remote request
- * @param queryParams - queryParams to be used.
- * @return RemoteSASKeyGenerationResponse
- */
- private RemoteSASKeyGenerationResponse makeRemoteRequest(String[] urls,
- String path, List queryParams)
- throws SASKeyGenerationException {
-
- try {
- String responseBody = remoteCallHelper
- .makeRemoteRequest(urls, path, queryParams, HttpGet.METHOD_NAME);
- return RESPONSE_READER.readValue(responseBody);
-
- } catch (WasbRemoteCallException remoteCallEx) {
- throw new SASKeyGenerationException("Encountered RemoteCallException"
- + " while retrieving SAS key from remote service", remoteCallEx);
- } catch (JsonParseException jsonParserEx) {
- throw new SASKeyGenerationException("Encountered JsonParseException "
- + "while parsing the response from remote"
- + " service into RemoteSASKeyGenerationResponse object",
- jsonParserEx);
- } catch (JsonMappingException jsonMappingEx) {
- throw new SASKeyGenerationException("Encountered JsonMappingException"
- + " while mapping the response from remote service into "
- + "RemoteSASKeyGenerationResponse object", jsonMappingEx);
- } catch (IOException ioEx) {
- throw new SASKeyGenerationException("Encountered IOException while "
- + "accessing remote service to retrieve SAS Key", ioEx);
- }
- }
-}
-
-/**
- * POJO representing the response expected from a Remote
- * SAS Key generation service.
- * The remote SAS Key generation service is expected to
- * return SAS key in json format:
- * {
- * "responseCode" : 0 or non-zero ,
- * "responseMessage" : relavant message on failure ,
- * "sasKey" : Requested SAS Key
- * }
- */
-class RemoteSASKeyGenerationResponse {
-
- /**
- * Response code for the call.
- */
- private int responseCode;
-
- /**
- * An intelligent message corresponding to
- * result. Specifically in case of failure
- * the reason for failure.
- */
- private String responseMessage;
-
- /**
- * SAS Key corresponding to the request.
- */
- private String sasKey;
-
- public int getResponseCode() {
- return responseCode;
- }
-
- public void setResponseCode(int responseCode) {
- this.responseCode = responseCode;
- }
-
- public String getResponseMessage() {
- return responseMessage;
- }
-
- public void setResponseMessage(String responseMessage) {
- this.responseMessage = responseMessage;
- }
-
- public String getSasKey() {
- return sasKey;
- }
-
- public void setSasKey(String sasKey) {
- this.sasKey = sasKey;
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
deleted file mode 100644
index eca8443b6c587..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URIBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.concurrent.TimeUnit;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE;
-
-/**
- * Class implementing WasbAuthorizerInterface using a remote
- * service that implements the authorization operation. This
- * class expects the url of the remote service to be passed
- * via config.
- */
-public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface {
-
- public static final Logger LOG = LoggerFactory
- .getLogger(RemoteWasbAuthorizerImpl.class);
- private static final ObjectReader RESPONSE_READER = new ObjectMapper()
- .readerFor(RemoteWasbAuthorizerResponse.class);
-
- /**
- * Configuration parameter name expected in the Configuration object to
- * provide the urls of the remote service instances. {@value}
- */
- public static final String KEY_REMOTE_AUTH_SERVICE_URLS =
- "fs.azure.authorization.remote.service.urls";
- /**
- * Authorization operation OP name in the remote service {@value}
- */
- private static final String CHECK_AUTHORIZATION_OP = "CHECK_AUTHORIZATION";
- /**
- * Query parameter specifying the access operation type. {@value}
- */
- private static final String ACCESS_OPERATION_QUERY_PARAM_NAME =
- "operation_type";
- /**
- * Query parameter specifying the wasb absolute path. {@value}
- */
- private static final String WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME =
- "wasb_absolute_path";
- /**
- * Query parameter name for sending owner of the specific resource {@value}
- */
- private static final String WASB_RESOURCE_OWNER_QUERY_PARAM_NAME =
- "wasb_resource_owner";
-
- /**
- * Authorization Remote http client retry policy enabled configuration key. {@value}
- */
- private static final String AUTHORIZER_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY =
- "fs.azure.authorizer.http.retry.policy.enabled";
-
- /**
- * Authorization Remote http client retry policy spec. {@value}
- */
- private static final String AUTHORIZER_HTTP_CLIENT_RETRY_POLICY_SPEC_SPEC =
- "fs.azure.authorizer.http.retry.policy.spec";
-
- /**
- * Authorization Remote http client retry policy spec default value. {@value}
- */
- private static final String AUTHORIZER_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
- "10,3,100,2";
-
- /**
- * Authorization caching period
- */
- private static final String AUTHORIZATION_CACHEENTRY_EXPIRY_PERIOD =
- "fs.azure.authorization.cacheentry.expiry.period";
-
- private WasbRemoteCallHelper remoteCallHelper = null;
- private boolean isKerberosSupportEnabled;
- private boolean isSpnegoTokenCacheEnabled;
- private RetryPolicy retryPolicy;
- private String[] commaSeparatedUrls = null;
- private CachingAuthorizer cache;
-
- @VisibleForTesting public void updateWasbRemoteCallHelper(
- WasbRemoteCallHelper helper) {
- this.remoteCallHelper = helper;
- }
-
- @Override
- public void init(Configuration conf)
- throws IOException {
- LOG.debug("Initializing RemoteWasbAuthorizerImpl instance");
- this.isKerberosSupportEnabled =
- conf.getBoolean(Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false);
- this.isSpnegoTokenCacheEnabled =
- conf.getBoolean(Constants.AZURE_ENABLE_SPNEGO_TOKEN_CACHE, true);
- this.commaSeparatedUrls =
- conf.getTrimmedStrings(KEY_REMOTE_AUTH_SERVICE_URLS);
- if (this.commaSeparatedUrls == null
- || this.commaSeparatedUrls.length <= 0) {
- throw new IOException(KEY_REMOTE_AUTH_SERVICE_URLS + " config not set"
- + " in configuration.");
- }
- this.retryPolicy = RetryUtils.getMultipleLinearRandomRetry(conf,
- AUTHORIZER_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY, true,
- AUTHORIZER_HTTP_CLIENT_RETRY_POLICY_SPEC_SPEC,
- AUTHORIZER_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT);
- if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled()) {
- this.remoteCallHelper = new SecureWasbRemoteCallHelper(retryPolicy, false,
- isSpnegoTokenCacheEnabled);
- } else {
- this.remoteCallHelper = new WasbRemoteCallHelper(retryPolicy);
- }
-
- this.cache = new CachingAuthorizer<>(
- conf.getTimeDuration(AUTHORIZATION_CACHEENTRY_EXPIRY_PERIOD, 5L, TimeUnit.MINUTES), "AUTHORIZATION"
- );
- this.cache.init(conf);
- }
-
- @Override
- public boolean authorize(String wasbAbsolutePath, String accessType, String resourceOwner)
- throws IOException {
-
- /* Make an exception for the internal -RenamePending files */
- if (wasbAbsolutePath.endsWith(NativeAzureFileSystem.FolderRenamePending.SUFFIX)) {
- return true;
- }
-
- CachedAuthorizerEntry cacheKey = new CachedAuthorizerEntry(wasbAbsolutePath, accessType, resourceOwner);
- Boolean cacheresult = cache.get(cacheKey);
- if (cacheresult != null) {
- return cacheresult;
- }
-
- boolean authorizeresult = authorizeInternal(wasbAbsolutePath, accessType, resourceOwner);
- cache.put(cacheKey, authorizeresult);
-
- return authorizeresult;
- }
-
- private boolean authorizeInternal(String wasbAbsolutePath, String accessType, String resourceOwner)
- throws IOException {
-
- try {
- final URIBuilder uriBuilder = new URIBuilder();
- uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP);
- uriBuilder
- .addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, wasbAbsolutePath);
- uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME, accessType);
- if (resourceOwner != null && StringUtils.isNotEmpty(resourceOwner)) {
- uriBuilder.addParameter(WASB_RESOURCE_OWNER_QUERY_PARAM_NAME,
- resourceOwner);
- }
-
- String responseBody = remoteCallHelper
- .makeRemoteRequest(commaSeparatedUrls, uriBuilder.getPath(),
- uriBuilder.getQueryParams(), HttpGet.METHOD_NAME);
-
- RemoteWasbAuthorizerResponse authorizerResponse = RESPONSE_READER
- .readValue(responseBody);
-
- if (authorizerResponse == null) {
- throw new WasbAuthorizationException(
- "RemoteWasbAuthorizerResponse object null from remote call");
- } else if (authorizerResponse.getResponseCode()
- == REMOTE_CALL_SUCCESS_CODE) {
- return authorizerResponse.getAuthorizationResult();
- } else {
- throw new WasbAuthorizationException(
- "Remote authorization" + " service encountered an error "
- + authorizerResponse.getResponseMessage());
- }
- } catch (WasbRemoteCallException | JsonParseException | JsonMappingException ex) {
- throw new WasbAuthorizationException(ex);
- }
- }
-}
-
-/**
- * POJO representing the response expected from a remote
- * authorization service.
- * The remote service is expected to return the authorization
- * response in the following JSON format
- * {
- * "responseCode" : 0 or non-zero ,
- * "responseMessage" : relevant message of failure
- * "authorizationResult" : authorization result
- * true - if auhorization allowed
- * false - otherwise.
- * }
- */
-class RemoteWasbAuthorizerResponse {
-
- private int responseCode;
- private boolean authorizationResult;
- private String responseMessage;
-
- public int getResponseCode() {
- return responseCode;
- }
-
- public void setResponseCode(int responseCode) {
- this.responseCode = responseCode;
- }
-
- public boolean getAuthorizationResult() {
- return authorizationResult;
- }
-
- public void setAuthorizationResult(boolean authorizationResult) {
- this.authorizationResult = authorizationResult;
- }
-
- public String getResponseMessage() {
- return responseMessage;
- }
-
- public void setResponseMessage(String message) {
- this.responseMessage = message;
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGenerationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGenerationException.java
deleted file mode 100644
index 7cfafc3ed9406..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGenerationException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * Exception that gets thrown during generation of SAS Key.
- *
- */
-public class SASKeyGenerationException extends AzureException {
-
- private static final long serialVersionUID = 1L;
-
- public SASKeyGenerationException(String message) {
- super(message);
- }
-
- public SASKeyGenerationException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public SASKeyGenerationException(Throwable t) {
- super(t);
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGeneratorImpl.java
deleted file mode 100644
index 1a8e7541fc86c..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGeneratorImpl.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Abstract base class for the SAS Key Generator implementation
- *
- */
-public abstract class SASKeyGeneratorImpl implements SASKeyGeneratorInterface {
-
- /**
- * Configuration key to be used to specify the expiry period for SAS keys
- * This value currently is specified in days. {@value}
- */
- public static final String KEY_SAS_KEY_EXPIRY_PERIOD =
- "fs.azure.sas.expiry.period";
-
- /**
- * Default value for the SAS key expiry period in days. {@value}
- */
- public static final long DEFAULT_CONTAINER_SAS_KEY_PERIOD = 90;
-
- private long sasKeyExpiryPeriod;
-
- private Configuration conf;
-
- public SASKeyGeneratorImpl(Configuration conf) {
- this.conf = conf;
- this.sasKeyExpiryPeriod = conf.getTimeDuration(
- KEY_SAS_KEY_EXPIRY_PERIOD, DEFAULT_CONTAINER_SAS_KEY_PERIOD,
- TimeUnit.DAYS);
- }
-
- public long getSasKeyExpiryPeriod() {
- return sasKeyExpiryPeriod;
- }
-
- public Configuration getConf() {
- return conf;
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGeneratorInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGeneratorInterface.java
deleted file mode 100644
index 3067c1096df84..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SASKeyGeneratorInterface.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.net.URI;
-
-/**
- * Iterface used by AzureNativeFileSysteStore to retrieve SAS Keys for the
- * respective azure storage entity. This interface is expected to be
- * implemented in two modes:
- * 1) Local Mode: In this mode SAS Keys are generated
- * in same address space as the WASB. This will be primarily used for
- * testing purposes.
- * 2) Remote Mode: In this mode SAS Keys are generated in a sepearte process
- * other than WASB and will be communicated via client.
- */
-public interface SASKeyGeneratorInterface {
-
- /**
- * Interface method to retrieve SAS Key for a container within the storage
- * account.
- *
- * @param accountName
- * - Storage account name
- * @param container
- * - Container name within the storage account.
- * @return SAS URI for the container.
- * @throws SASKeyGenerationException Exception that gets thrown during
- * generation of SAS Key.
- */
- URI getContainerSASUri(String accountName, String container)
- throws SASKeyGenerationException;
-
- /**
- * Interface method to retrieve SAS Key for a blob within the container of the
- * storage account.
- *
- * @param accountName
- * - Storage account name
- * @param container
- * - Container name within the storage account.
- * @param relativePath
- * - Relative path within the container
- * @return SAS URI for the relative path blob.
- * @throws SASKeyGenerationException Exception that gets thrown during
- * generation of SAS Key.
- */
- URI getRelativeBlobSASUri(String accountName, String container,
- String relativePath) throws SASKeyGenerationException;
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureModeException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureModeException.java
deleted file mode 100644
index 5bec77d165015..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureModeException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * Exception that is thrown when any error is encountered
- * is SAS Mode operation of WASB.
- */
-public class SecureModeException extends AzureException {
-
- private static final long serialVersionUID = 1L;
-
- public SecureModeException(String message) {
- super(message);
- }
-
- public SecureModeException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public SecureModeException(Throwable t) {
- super(t);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
deleted file mode 100644
index f6eb75cad59c8..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
+++ /dev/null
@@ -1,606 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RetryPolicyFactory;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.StorageUri;
-import com.microsoft.azure.storage.blob.BlobProperties;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.BlockListingFilter;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import com.microsoft.azure.storage.blob.CloudPageBlob;
-import com.microsoft.azure.storage.blob.CopyState;
-import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
-import com.microsoft.azure.storage.blob.ListBlobItem;
-import com.microsoft.azure.storage.blob.BlobListingDetails;
-import com.microsoft.azure.storage.blob.PageRange;
-import com.microsoft.azure.storage.blob.BlockEntry;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/***
- * An implementation of the StorageInterface for SAS Key mode.
- *
- */
-
-public class SecureStorageInterfaceImpl extends StorageInterface {
-
- public static final Logger LOG = LoggerFactory.getLogger(
- SecureStorageInterfaceImpl.class);
- public static final String SAS_ERROR_CODE = "SAS Error";
- private SASKeyGeneratorInterface sasKeyGenerator;
- private String storageAccount;
- private RetryPolicyFactory retryPolicy;
- private int timeoutIntervalInMs;
- private boolean useContainerSasKeyForAllAccess;
-
- /**
- * Configuration key to specify if containerSasKey should be used for all accesses
- */
- public static final String KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS =
- "fs.azure.saskey.usecontainersaskeyforallaccess";
-
- public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode,
- Configuration conf) throws SecureModeException {
-
- if (useLocalSASKeyMode) {
- LOG.debug("Authenticating with SecureStorage and local SAS key");
- this.sasKeyGenerator = new LocalSASKeyGeneratorImpl(conf);
- } else {
- LOG.debug("Authenticating with SecureStorage and remote SAS key generation");
- RemoteSASKeyGeneratorImpl remoteSasKeyGenerator =
- new RemoteSASKeyGeneratorImpl(conf);
- try {
- remoteSasKeyGenerator.initialize(conf);
- } catch (IOException ioe) {
- throw new SecureModeException("Remote SAS Key mode could"
- + " not be initialized", ioe);
- }
- this.sasKeyGenerator = remoteSasKeyGenerator;
- }
- this.useContainerSasKeyForAllAccess = conf.getBoolean(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, true);
- LOG.debug("Container SAS key {} be used for all access",
- useContainerSasKeyForAllAccess ? "will" : "will not");
- }
-
- @Override
- public void setTimeoutInMs(int timeoutInMs) {
- timeoutIntervalInMs = timeoutInMs;
- }
-
- @Override
- public void setRetryPolicyFactory(RetryPolicyFactory retryPolicyFactory) {
- retryPolicy = retryPolicyFactory;
- }
-
- @Override
- public void createBlobClient(CloudStorageAccount account) {
- String errorMsg = "createBlobClient is an invalid operation in"
- + " SAS Key Mode";
- LOG.error(errorMsg);
- throw new UnsupportedOperationException(errorMsg);
- }
-
- @Override
- public void createBlobClient(URI baseUri) {
- String errorMsg = "createBlobClient is an invalid operation in "
- + "SAS Key Mode";
- LOG.error(errorMsg);
- throw new UnsupportedOperationException(errorMsg);
- }
-
- @Override
- public void createBlobClient(URI baseUri, StorageCredentials credentials) {
- String errorMsg = "createBlobClient is an invalid operation in SAS "
- + "Key Mode";
- LOG.error(errorMsg);
- throw new UnsupportedOperationException(errorMsg);
- }
-
- @Override
- public StorageCredentials getCredentials() {
- String errorMsg = "getCredentials is an invalid operation in SAS "
- + "Key Mode";
- LOG.error(errorMsg);
- throw new UnsupportedOperationException(errorMsg);
- }
-
- @Override
- public CloudBlobContainerWrapper getContainerReference(String name)
- throws URISyntaxException, StorageException {
-
- try {
- CloudBlobContainer container = new CloudBlobContainer(sasKeyGenerator.getContainerSASUri(
- storageAccount, name));
- if (retryPolicy != null) {
- container.getServiceClient().getDefaultRequestOptions().setRetryPolicyFactory(retryPolicy);
- }
- if (timeoutIntervalInMs > 0) {
- container.getServiceClient().getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs);
- }
- return (useContainerSasKeyForAllAccess)
- ? new SASCloudBlobContainerWrapperImpl(storageAccount, container, null)
- : new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator);
- } catch (SASKeyGenerationException sasEx) {
- String errorMsg = "Encountered SASKeyGeneration exception while "
- + "generating SAS Key for container : " + name
- + " inside Storage account : " + storageAccount;
- LOG.error(errorMsg);
- throw new StorageException(SAS_ERROR_CODE, errorMsg, sasEx);
- }
- }
-
- public void setStorageAccountName(String storageAccount) {
- this.storageAccount = storageAccount;
- }
-
- @InterfaceAudience.Private
- static class SASCloudBlobContainerWrapperImpl
- extends CloudBlobContainerWrapper {
-
- private final CloudBlobContainer container;
- private String storageAccount;
- private SASKeyGeneratorInterface sasKeyGenerator;
-
- public SASCloudBlobContainerWrapperImpl(String storageAccount,
- CloudBlobContainer container, SASKeyGeneratorInterface sasKeyGenerator) {
- this.storageAccount = storageAccount;
- this.container = container;
- this.sasKeyGenerator = sasKeyGenerator;
- }
-
- @Override
- public String getName() {
- return container.getName();
- }
-
- @Override
- public boolean exists(OperationContext opContext) throws StorageException {
- return container.exists(AccessCondition.generateEmptyCondition(), null,
- opContext);
- }
-
- @Override
- public void create(OperationContext opContext) throws StorageException {
- container.create(null, opContext);
- }
-
- @Override
- public HashMap getMetadata() {
- return container.getMetadata();
- }
-
- @Override
- public void setMetadata(HashMap metadata) {
- container.setMetadata(metadata);
- }
-
- @Override
- public void downloadAttributes(OperationContext opContext)
- throws StorageException {
- container.downloadAttributes(AccessCondition.generateEmptyCondition(),
- null, opContext);
- }
-
- @Override
- public void uploadMetadata(OperationContext opContext)
- throws StorageException {
- container.uploadMetadata(AccessCondition.generateEmptyCondition(), null,
- opContext);
- }
-
- @Override
- public CloudBlobDirectoryWrapper getDirectoryReference(String relativePath)
- throws URISyntaxException, StorageException {
-
- CloudBlobDirectory dir = container.getDirectoryReference(relativePath);
- return new SASCloudBlobDirectoryWrapperImpl(dir);
- }
-
- @Override
- public CloudBlobWrapper getBlockBlobReference(String relativePath)
- throws URISyntaxException, StorageException {
- try {
- CloudBlockBlob blob = (sasKeyGenerator!=null)
- ? new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(), relativePath))
- : container.getBlockBlobReference(relativePath);
- blob.getServiceClient().setDefaultRequestOptions(
- container.getServiceClient().getDefaultRequestOptions());
- return new SASCloudBlockBlobWrapperImpl(blob);
- } catch (SASKeyGenerationException sasEx) {
- String errorMsg = "Encountered SASKeyGeneration exception while "
- + "generating SAS Key for relativePath : " + relativePath
- + " inside container : " + getName() + " Storage account : " + storageAccount;
- LOG.error(errorMsg);
- throw new StorageException(SAS_ERROR_CODE, errorMsg, sasEx);
- }
- }
-
- @Override
- public CloudBlobWrapper getPageBlobReference(String relativePath)
- throws URISyntaxException, StorageException {
- try {
- CloudPageBlob blob = (sasKeyGenerator!=null)
- ? new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(storageAccount, getName(), relativePath))
- : container.getPageBlobReference(relativePath);
-
- blob.getServiceClient().setDefaultRequestOptions(
- container.getServiceClient().getDefaultRequestOptions());
- return new SASCloudPageBlobWrapperImpl(blob);
- } catch (SASKeyGenerationException sasEx) {
- String errorMsg = "Encountered SASKeyGeneration exception while "
- + "generating SAS Key for relativePath : " + relativePath
- + " inside container : " + getName()
- + " Storage account : " + storageAccount;
- LOG.error(errorMsg);
- throw new StorageException(SAS_ERROR_CODE, errorMsg, sasEx);
- }
- }
- }
-
- //
- // WrappingIterator
- //
-
- /**
- * This iterator wraps every ListBlobItem as they come from the listBlobs()
- * calls to their proper wrapping objects.
- */
- private static class SASWrappingIterator implements Iterator {
- private final Iterator present;
-
- public SASWrappingIterator(Iterator present) {
- this.present = present;
- }
-
- public static Iterable wrap(
- final Iterable present) {
- return new Iterable() {
- @Override
- public Iterator iterator() {
- return new SASWrappingIterator(present.iterator());
- }
- };
- }
-
- @Override
- public boolean hasNext() {
- return present.hasNext();
- }
-
- @Override
- public ListBlobItem next() {
- ListBlobItem unwrapped = present.next();
- if (unwrapped instanceof CloudBlobDirectory) {
- return new SASCloudBlobDirectoryWrapperImpl((CloudBlobDirectory) unwrapped);
- } else if (unwrapped instanceof CloudBlockBlob) {
- return new SASCloudBlockBlobWrapperImpl((CloudBlockBlob) unwrapped);
- } else if (unwrapped instanceof CloudPageBlob) {
- return new SASCloudPageBlobWrapperImpl((CloudPageBlob) unwrapped);
- } else {
- return unwrapped;
- }
- }
-
- @Override
- public void remove() {
- present.remove();
- }
- }
-
- //
- // CloudBlobDirectoryWrapperImpl
- //
- @InterfaceAudience.Private
- static class SASCloudBlobDirectoryWrapperImpl extends CloudBlobDirectoryWrapper {
- private final CloudBlobDirectory directory;
-
- public SASCloudBlobDirectoryWrapperImpl(CloudBlobDirectory directory) {
- this.directory = directory;
- }
-
- @Override
- public URI getUri() {
- return directory.getUri();
- }
-
- @Override
- public Iterable listBlobs(String prefix,
- boolean useFlatBlobListing, EnumSet listingDetails,
- BlobRequestOptions options, OperationContext opContext)
- throws URISyntaxException, StorageException {
- return SASWrappingIterator.wrap(directory.listBlobs(prefix,
- useFlatBlobListing, listingDetails, options, opContext));
- }
-
- @Override
- public CloudBlobContainer getContainer() throws URISyntaxException,
- StorageException {
- return directory.getContainer();
- }
-
- @Override
- public CloudBlobDirectory getParent() throws URISyntaxException,
- StorageException {
- return directory.getParent();
- }
-
- @Override
- public StorageUri getStorageUri() {
- return directory.getStorageUri();
- }
- }
-
- abstract static class SASCloudBlobWrapperImpl implements CloudBlobWrapper {
- private final CloudBlob blob;
- @Override
- public CloudBlob getBlob() {
- return blob;
- }
-
- public URI getUri() {
- return getBlob().getUri();
- }
-
- protected SASCloudBlobWrapperImpl(CloudBlob blob) {
- this.blob = blob;
- }
-
- @Override
- public HashMap getMetadata() {
- return getBlob().getMetadata();
- }
-
- @Override
- public void delete(OperationContext opContext, SelfRenewingLease lease)
- throws StorageException {
- getBlob().delete(DeleteSnapshotsOption.NONE, getLeaseCondition(lease),
- null, opContext);
- }
-
- /**
- * Return and access condition for this lease, or else null if
- * there's no lease.
- */
- private AccessCondition getLeaseCondition(SelfRenewingLease lease) {
- AccessCondition leaseCondition = null;
- if (lease != null) {
- leaseCondition = AccessCondition.generateLeaseCondition(lease.getLeaseID());
- }
- return leaseCondition;
- }
-
- @Override
- public boolean exists(OperationContext opContext)
- throws StorageException {
- return getBlob().exists(null, null, opContext);
- }
-
- @Override
- public void downloadAttributes(
- OperationContext opContext) throws StorageException {
- getBlob().downloadAttributes(null, null, opContext);
- }
-
- @Override
- public BlobProperties getProperties() {
- return getBlob().getProperties();
- }
-
- @Override
- public void setMetadata(HashMap metadata) {
- getBlob().setMetadata(metadata);
- }
-
- @Override
- public InputStream openInputStream(
- BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- return getBlob().openInputStream(null, options, opContext);
- }
-
- public OutputStream openOutputStream(
- BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- return ((CloudBlockBlob) getBlob()).openOutputStream(null, options, opContext);
- }
-
- public void upload(InputStream sourceStream, OperationContext opContext)
- throws StorageException, IOException {
- getBlob().upload(sourceStream, 0, null, null, opContext);
- }
-
- @Override
- public CloudBlobContainer getContainer() throws URISyntaxException,
- StorageException {
- return getBlob().getContainer();
- }
-
- @Override
- public CloudBlobDirectory getParent() throws URISyntaxException,
- StorageException {
- return getBlob().getParent();
- }
-
- @Override
- public void uploadMetadata(OperationContext opContext)
- throws StorageException {
- uploadMetadata(null, null, opContext);
- }
-
- @Override
- public void uploadMetadata(AccessCondition accessConditions, BlobRequestOptions options,
- OperationContext opContext) throws StorageException{
- getBlob().uploadMetadata(accessConditions, options, opContext);
- }
-
- public void uploadProperties(OperationContext opContext, SelfRenewingLease lease)
- throws StorageException {
-
- // Include lease in request if lease not null.
- getBlob().uploadProperties(getLeaseCondition(lease), null, opContext);
- }
-
- @Override
- public int getStreamMinimumReadSizeInBytes() {
- return getBlob().getStreamMinimumReadSizeInBytes();
- }
-
- @Override
- public void setStreamMinimumReadSizeInBytes(int minimumReadSizeBytes) {
- getBlob().setStreamMinimumReadSizeInBytes(minimumReadSizeBytes);
- }
-
- @Override
- public void setWriteBlockSizeInBytes(int writeBlockSizeBytes) {
- getBlob().setStreamWriteSizeInBytes(writeBlockSizeBytes);
- }
-
- @Override
- public StorageUri getStorageUri() {
- return getBlob().getStorageUri();
- }
-
- @Override
- public CopyState getCopyState() {
- return getBlob().getCopyState();
- }
-
- @Override
- public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
- OperationContext opContext, boolean overwriteDestination)
- throws StorageException, URISyntaxException {
- AccessCondition dstAccessCondition =
- overwriteDestination
- ? null
- : AccessCondition.generateIfNotExistsCondition();
- getBlob().startCopy(sourceBlob.getBlob().getQualifiedUri(),
- null, dstAccessCondition, options, opContext);
- }
-
- @Override
- public void downloadRange(long offset, long length, OutputStream outStream,
- BlobRequestOptions options, OperationContext opContext)
- throws StorageException, IOException {
-
- getBlob().downloadRange(offset, length, outStream, null, options, opContext);
- }
-
- @Override
- public SelfRenewingLease acquireLease() throws StorageException {
- return new SelfRenewingLease(this, false);
- }
- }
-
- //
- // CloudBlockBlobWrapperImpl
- //
-
- static class SASCloudBlockBlobWrapperImpl extends SASCloudBlobWrapperImpl implements CloudBlockBlobWrapper {
-
- public SASCloudBlockBlobWrapperImpl(CloudBlockBlob blob) {
- super(blob);
- }
-
- public OutputStream openOutputStream(
- BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- return ((CloudBlockBlob) getBlob()).openOutputStream(null, options, opContext);
- }
-
- public void upload(InputStream sourceStream, OperationContext opContext)
- throws StorageException, IOException {
- getBlob().upload(sourceStream, 0, null, null, opContext);
- }
-
- public void uploadProperties(OperationContext opContext)
- throws StorageException {
- getBlob().uploadProperties(null, null, opContext);
- }
-
- @Override
- public List downloadBlockList(BlockListingFilter filter, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException {
- return ((CloudBlockBlob) getBlob()).downloadBlockList(filter, null, options, opContext);
-
- }
-
- @Override
- public void uploadBlock(String blockId, AccessCondition accessCondition,
- InputStream sourceStream,
- long length, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException {
- ((CloudBlockBlob) getBlob()).uploadBlock(blockId, sourceStream, length,
- accessCondition, options, opContext);
- }
-
- @Override
- public void commitBlockList(List blockList, AccessCondition accessCondition, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException {
- ((CloudBlockBlob) getBlob()).commitBlockList(blockList, accessCondition, options, opContext);
- }
- }
-
- static class SASCloudPageBlobWrapperImpl extends SASCloudBlobWrapperImpl implements CloudPageBlobWrapper {
- public SASCloudPageBlobWrapperImpl(CloudPageBlob blob) {
- super(blob);
- }
-
- public void create(final long length, BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- ((CloudPageBlob) getBlob()).create(length, null, options, opContext);
- }
-
- public void uploadPages(final InputStream sourceStream, final long offset,
- final long length, BlobRequestOptions options, OperationContext opContext)
- throws StorageException, IOException {
- ((CloudPageBlob) getBlob()).uploadPages(sourceStream, offset, length, null,
- options, opContext);
- }
-
- public ArrayList downloadPageRanges(BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- return ((CloudPageBlob) getBlob()).downloadPageRanges(
- null, options, opContext);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
deleted file mode 100644
index f4ec1721ec490..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.commons.lang3.Validate;
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.fs.azure.security.SpnegoToken;
-import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.client.Authenticator;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URIBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URISyntaxException;
-import java.security.PrivilegedExceptionAction;
-import java.util.List;
-
-/**
- * Helper class the has constants and helper methods
- * used in WASB when integrating with a remote http cred
- * service which uses Kerberos and delegation tokens.
- * Currently, remote service will be used to generate
- * SAS keys, authorization and delegation token operations.
- */
-public class SecureWasbRemoteCallHelper extends WasbRemoteCallHelper {
-
- public static final Logger LOG =
- LoggerFactory.getLogger(SecureWasbRemoteCallHelper.class);
- /**
- * Delegation token query parameter to be used when making rest call.
- */
- private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME = "delegation";
-
- /**
- * Delegation token to be used for making the remote call.
- */
- private Token delegationToken = null;
-
- /**
- * Does Remote Http Call requires Kerberos Authentication always, even if the delegation token is present.
- */
- private boolean alwaysRequiresKerberosAuth;
-
- /**
- * Enable caching of Spnego token.
- */
- private boolean isSpnegoTokenCachingEnabled;
-
- /**
- * Cached SPNEGO token.
- */
- private SpnegoToken spnegoToken;
-
- public SecureWasbRemoteCallHelper(RetryPolicy retryPolicy,
- boolean alwaysRequiresKerberosAuth, boolean isSpnegoTokenCachingEnabled) {
- super(retryPolicy);
- this.alwaysRequiresKerberosAuth = alwaysRequiresKerberosAuth;
- this.isSpnegoTokenCachingEnabled = isSpnegoTokenCachingEnabled;
- }
-
- @Override
- public String makeRemoteRequest(final String[] urls,
- final String path, final List queryParams,
- final String httpMethod) throws IOException {
- final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- UserGroupInformation connectUgi = ugi.getRealUser();
- if (connectUgi != null) {
- queryParams.add(new NameValuePair() {
- @Override public String getName() {
- return Constants.DOAS_PARAM;
- }
-
- @Override public String getValue() {
- return ugi.getShortUserName();
- }
- });
- } else {
- connectUgi = ugi;
- }
-
- final Token delegationToken = getDelegationToken(ugi);
- if (!alwaysRequiresKerberosAuth && delegationToken != null) {
- final String delegationTokenEncodedUrlString =
- delegationToken.encodeToUrlString();
- queryParams.add(new NameValuePair() {
- @Override public String getName() {
- return DELEGATION_TOKEN_QUERY_PARAM_NAME;
- }
-
- @Override public String getValue() {
- return delegationTokenEncodedUrlString;
- }
- });
- }
-
- if (delegationToken == null) {
- connectUgi.checkTGTAndReloginFromKeytab();
- }
- String s = null;
- try {
- s = connectUgi.doAs(new PrivilegedExceptionAction() {
- @Override public String run() throws Exception {
- return retryableRequest(urls, path, queryParams, httpMethod);
- }
- });
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IOException(e.getMessage(), e);
- }
- return s;
- }
-
- @Override
- public HttpUriRequest getHttpRequest(String[] urls, String path,
- List queryParams, int urlIndex, String httpMethod,
- boolean requiresNewAuth) throws URISyntaxException, IOException {
- URIBuilder uriBuilder =
- new URIBuilder(urls[urlIndex]).setPath(path).setParameters(queryParams);
- if (uriBuilder.getHost().equals("localhost")) {
- uriBuilder.setHost(InetAddress.getLocalHost().getCanonicalHostName());
- }
- HttpUriRequest httpUriRequest = null;
- switch (httpMethod) {
- case HttpPut.METHOD_NAME:
- httpUriRequest = new HttpPut(uriBuilder.build());
- break;
- case HttpPost.METHOD_NAME:
- httpUriRequest = new HttpPost(uriBuilder.build());
- break;
- default:
- httpUriRequest = new HttpGet(uriBuilder.build());
- break;
- }
-
- LOG.debug("SecureWasbRemoteCallHelper#getHttpRequest() {}",
- uriBuilder.build().toURL());
- if (alwaysRequiresKerberosAuth || delegationToken == null) {
- AuthenticatedURL.Token token = null;
- final Authenticator kerberosAuthenticator =
- new KerberosDelegationTokenAuthenticator();
- try {
- if (isSpnegoTokenCachingEnabled && !requiresNewAuth
- && spnegoToken != null && spnegoToken.isTokenValid()){
- token = spnegoToken.getToken();
- } else {
- token = new AuthenticatedURL.Token();
- kerberosAuthenticator.authenticate(uriBuilder.build().toURL(), token);
- spnegoToken = new SpnegoToken(token);
- }
- } catch (AuthenticationException e) {
- throw new WasbRemoteCallException(
- Constants.AUTHENTICATION_FAILED_ERROR_MESSAGE, e);
- }
- Validate.isTrue(token.isSet(),
- "Authenticated Token is NOT present. The request cannot proceed.");
-
- httpUriRequest.setHeader("Cookie",
- AuthenticatedURL.AUTH_COOKIE + "=" + token);
- }
- return httpUriRequest;
- }
-
- private Token getDelegationToken(
- UserGroupInformation userGroupInformation) throws IOException {
- if (this.delegationToken == null) {
- Token token = null;
- for (Token iterToken : userGroupInformation.getTokens()) {
- if (iterToken.getKind()
- .equals(WasbDelegationTokenIdentifier.TOKEN_KIND)) {
- token = iterToken;
- LOG.debug("{} token found in cache : {}",
- WasbDelegationTokenIdentifier.TOKEN_KIND, iterToken);
- break;
- }
- }
- LOG.debug("UGI Information: {}", userGroupInformation.toString());
-
- // ugi tokens are usually indicative of a task which can't
- // refetch tokens. even if ugi has credentials, don't attempt
- // to get another token to match hdfs/rpc behavior
- if (token != null) {
- LOG.debug("Using UGI token: {}", token);
- setDelegationToken(token);
- }
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Delegation token from cache - {}", delegationToken != null
- ? delegationToken.encodeToUrlString()
- : "null");
- }
- return this.delegationToken;
- }
-
- private void setDelegationToken(
- final Token token) {
- synchronized (this) {
- this.delegationToken = token;
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
deleted file mode 100644
index 8ab568fdc3bfc..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
-
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static com.microsoft.azure.storage.StorageErrorCodeStrings.LEASE_ALREADY_PRESENT;
-
-/**
- * An Azure blob lease that automatically renews itself indefinitely
- * using a background thread. Use it to synchronize distributed processes,
- * or to prevent writes to the blob by other processes that don't
- * have the lease.
- *
- * Creating a new Lease object blocks the caller until the Azure blob lease is
- * acquired.
- *
- * Attempting to get a lease on a non-existent blob throws StorageException.
- *
- * Call free() to release the Lease.
- *
- * You can use this Lease like a distributed lock. If the holder process
- * dies, the lease will time out since it won't be renewed.
- */
-public class SelfRenewingLease {
-
- private CloudBlobWrapper blobWrapper;
- private Thread renewer;
- private volatile boolean leaseFreed;
- private String leaseID = null;
- private static final int LEASE_TIMEOUT = 60; // Lease timeout in seconds
-
- // Time to wait to renew lease in milliseconds
- public static final int LEASE_RENEWAL_PERIOD = 40000;
- private static final Logger LOG = LoggerFactory.getLogger(SelfRenewingLease.class);
-
- // Used to allocate thread serial numbers in thread name
- private static AtomicInteger threadNumber = new AtomicInteger(0);
-
-
- // Time to wait to retry getting the lease in milliseconds
- @VisibleForTesting
- static final int LEASE_ACQUIRE_RETRY_INTERVAL = 2000;
-
- public SelfRenewingLease(CloudBlobWrapper blobWrapper, boolean throwIfPresent)
- throws StorageException {
-
- this.leaseFreed = false;
- this.blobWrapper = blobWrapper;
-
- // Keep trying to get the lease until you get it.
- CloudBlob blob = blobWrapper.getBlob();
- while(leaseID == null) {
- try {
- leaseID = blob.acquireLease(LEASE_TIMEOUT, null);
- } catch (StorageException e) {
-
- if (throwIfPresent && e.getErrorCode().equals(LEASE_ALREADY_PRESENT)) {
- throw e;
- }
-
- // Throw again if we don't want to keep waiting.
- // We expect it to be that the lease is already present,
- // or in some cases that the blob does not exist.
- if (!LEASE_ALREADY_PRESENT.equals(e.getErrorCode())) {
- LOG.info(
- "Caught exception when trying to get lease on blob "
- + blobWrapper.getUri().toString() + ". " + e.getMessage());
- throw e;
- }
- }
- if (leaseID == null) {
- try {
- Thread.sleep(LEASE_ACQUIRE_RETRY_INTERVAL);
- } catch (InterruptedException e) {
-
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- }
- }
- }
- renewer = new SubjectInheritingThread(new Renewer());
-
- // A Renewer running should not keep JVM from exiting, so make it a daemon.
- renewer.setDaemon(true);
- renewer.setName("AzureLeaseRenewer-" + threadNumber.getAndIncrement());
- renewer.start();
- LOG.debug("Acquired lease " + leaseID + " on " + blob.getUri()
- + " managed by thread " + renewer.getName());
- }
-
- /**
- * Free the lease and stop the keep-alive thread.
- * @throws StorageException Thrown when fail to free the lease.
- */
- public void free() throws StorageException {
- AccessCondition accessCondition = AccessCondition.generateEmptyCondition();
- accessCondition.setLeaseID(leaseID);
- try {
- blobWrapper.getBlob().releaseLease(accessCondition);
- } catch (StorageException e) {
- if ("BlobNotFound".equals(e.getErrorCode())) {
-
- // Don't do anything -- it's okay to free a lease
- // on a deleted file. The delete freed the lease
- // implicitly.
- } else {
-
- // This error is not anticipated, so re-throw it.
- LOG.warn("Unanticipated exception when trying to free lease " + leaseID
- + " on " + blobWrapper.getStorageUri());
- throw(e);
- }
- } finally {
-
- // Even if releasing the lease fails (e.g. because the file was deleted),
- // make sure to record that we freed the lease, to terminate the
- // keep-alive thread.
- leaseFreed = true;
- LOG.debug("Freed lease " + leaseID + " on " + blobWrapper.getUri()
- + " managed by thread " + renewer.getName());
- }
- }
-
- public boolean isFreed() {
- return leaseFreed;
- }
-
- public String getLeaseID() {
- return leaseID;
- }
-
- public CloudBlob getCloudBlob() {
- return blobWrapper.getBlob();
- }
-
- private class Renewer implements Runnable {
-
- /**
- * Start a keep-alive thread that will continue to renew
- * the lease until it is freed or the process dies.
- */
- @Override
- public void run() {
- LOG.debug("Starting lease keep-alive thread.");
- AccessCondition accessCondition =
- AccessCondition.generateEmptyCondition();
- accessCondition.setLeaseID(leaseID);
-
- while(!leaseFreed) {
- try {
- Thread.sleep(LEASE_RENEWAL_PERIOD);
- } catch (InterruptedException e) {
- LOG.debug("Keep-alive thread for lease " + leaseID +
- " interrupted.");
-
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- }
- try {
- if (!leaseFreed) {
- blobWrapper.getBlob().renewLease(accessCondition);
-
- // It'll be very rare to renew the lease (most will be short)
- // so log that we did it, to help with system debugging.
- LOG.info("Renewed lease " + leaseID + " on "
- + getCloudBlob().getUri());
- }
- } catch (StorageException e) {
- if (!leaseFreed) {
-
- // Free the lease so we don't leave this thread running forever.
- leaseFreed = true;
-
- // Normally leases should be freed and there should be no
- // exceptions, so log a warning.
- LOG.warn("Attempt to renew lease " + leaseID + " on "
- + getCloudBlob().getUri()
- + " failed, but lease not yet freed. Reason: " +
- e.getMessage());
- }
- }
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java
deleted file mode 100644
index ad71016a745e7..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.net.HttpURLConnection;
-import java.util.Date;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RequestResult;
-import com.microsoft.azure.storage.ResponseReceivedEvent;
-import com.microsoft.azure.storage.SendingRequestEvent;
-import com.microsoft.azure.storage.StorageEvent;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/*
- * Self throttling is implemented by hooking into send & response callbacks
- * One instance of this class is created per operationContext so each blobUpload/blobDownload/etc.
- *
- * Self throttling only applies to 2nd and subsequent packets of an operation. This is a simple way to
- * ensure it only affects bulk transfers and not every tiny request.
- *
- * A blobDownload will involve sequential packet transmissions and so there are no concurrency concerns
- * A blobUpload will generally involve concurrent upload worker threads that share one operationContext and one throttling instance.
- * -- we do not track the latencies for each worker thread as they are doing similar work and will rarely collide in practice.
- * -- concurrent access to lastE2Edelay must be protected.
- * -- volatile is necessary and should be sufficient to protect simple access to primitive values (java 1.5 onwards)
- * -- synchronized{} blocks are also used to be conservative and for easier maintenance.
- *
- * If an operation were to perform concurrent GETs and PUTs there is the possibility of getting confused regarding
- * whether lastE2Edelay was a read or write measurement. This scenario does not occur.
- *
- * readFactor = target read throughput as factor of unrestricted throughput.
- * writeFactor = target write throughput as factor of unrestricted throughput.
- *
- * As we introduce delays it is important to only measure the actual E2E latency and not the augmented latency
- * To achieve this, we fiddle the 'startDate' of the transfer tracking object.
- */
-
-
-/**
- *
- * Introduces delays in our Azure traffic to prevent overrunning the server-side throttling limits.
- *
- */
-@InterfaceAudience.Private
-public class SelfThrottlingIntercept {
- public static final Logger LOG = LoggerFactory.getLogger(SelfThrottlingIntercept.class);
-
- private final float readFactor;
- private final float writeFactor;
- private final OperationContext operationContext;
-
- // Concurrency: access to non-final members must be thread-safe
- private long lastE2Elatency;
-
- public SelfThrottlingIntercept(OperationContext operationContext,
- float readFactor, float writeFactor) {
- this.operationContext = operationContext;
- this.readFactor = readFactor;
- this.writeFactor = writeFactor;
- }
-
- public static void hook(OperationContext operationContext, float readFactor,
- float writeFactor) {
-
- SelfThrottlingIntercept throttler = new SelfThrottlingIntercept(
- operationContext, readFactor, writeFactor);
- ResponseReceivedListener responseListener = throttler.new ResponseReceivedListener();
- SendingRequestListener sendingListener = throttler.new SendingRequestListener();
-
- operationContext.getResponseReceivedEventHandler().addListener(
- responseListener);
- operationContext.getSendingRequestEventHandler().addListener(
- sendingListener);
- }
-
- public void responseReceived(ResponseReceivedEvent event) {
- RequestResult result = event.getRequestResult();
- Date startDate = result.getStartDate();
- Date stopDate = result.getStopDate();
- long elapsed = stopDate.getTime() - startDate.getTime();
-
- synchronized (this) {
- this.lastE2Elatency = elapsed;
- }
-
- if (LOG.isDebugEnabled()) {
- int statusCode = result.getStatusCode();
- String etag = result.getEtag();
- HttpURLConnection urlConnection = (HttpURLConnection) event
- .getConnectionObject();
- int contentLength = urlConnection.getContentLength();
- String requestMethod = urlConnection.getRequestMethod();
- long threadId = Thread.currentThread().getId();
- LOG.debug(String
- .format(
- "SelfThrottlingIntercept:: ResponseReceived: threadId=%d, Status=%d, Elapsed(ms)=%d, ETAG=%s, contentLength=%d, requestMethod=%s",
- threadId, statusCode, elapsed, etag, contentLength, requestMethod));
- }
- }
-
- public void sendingRequest(SendingRequestEvent sendEvent) {
- long lastLatency;
- boolean operationIsRead; // for logging
- synchronized (this) {
-
- lastLatency = this.lastE2Elatency;
- }
-
- float sleepMultiple;
- HttpURLConnection urlConnection = (HttpURLConnection) sendEvent
- .getConnectionObject();
-
- // Azure REST API never uses POST, so PUT is a sufficient test for an
- // upload.
- if (urlConnection.getRequestMethod().equalsIgnoreCase("PUT")) {
- operationIsRead = false;
- sleepMultiple = (1 / writeFactor) - 1;
- } else {
- operationIsRead = true;
- sleepMultiple = (1 / readFactor) - 1;
- }
-
- long sleepDuration = (long) (sleepMultiple * lastLatency);
- if (sleepDuration < 0) {
- sleepDuration = 0;
- }
-
- if (sleepDuration > 0) {
- try {
- // Thread.sleep() is not exact but it seems sufficiently accurate for
- // our needs. If needed this could become a loop of small waits that
- // tracks actual
- // elapsed time.
- Thread.sleep(sleepDuration);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
-
- // reset to avoid counting the sleep against request latency
- sendEvent.getRequestResult().setStartDate(new Date());
- }
-
- if (LOG.isDebugEnabled()) {
- boolean isFirstRequest = (lastLatency == 0);
- long threadId = Thread.currentThread().getId();
- LOG.debug(String
- .format(
- " SelfThrottlingIntercept:: SendingRequest: threadId=%d, requestType=%s, isFirstRequest=%b, sleepDuration=%d",
- threadId, operationIsRead ? "read " : "write", isFirstRequest,
- sleepDuration));
- }
- }
-
- // simply forwards back to the main class.
- // this is necessary as our main class cannot implement two base-classes.
- @InterfaceAudience.Private
- class SendingRequestListener extends StorageEvent {
-
- @Override
- public void eventOccurred(SendingRequestEvent event) {
- sendingRequest(event);
- }
- }
-
- // simply forwards back to the main class.
- // this is necessary as our main class cannot implement two base-classes.
- @InterfaceAudience.Private
- class ResponseReceivedListener extends StorageEvent {
-
- @Override
- public void eventOccurred(ResponseReceivedEvent event) {
- responseReceived(event);
- }
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java
deleted file mode 100644
index 98f9de7bff554..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.net.HttpURLConnection;
-import java.security.InvalidKeyException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import com.microsoft.azure.storage.Constants.HeaderConstants;
-import com.microsoft.azure.storage.core.StorageCredentialsHelper;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.SendingRequestEvent;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageEvent;
-import com.microsoft.azure.storage.StorageException;
-
-/**
- * Manages the lifetime of binding on the operation contexts to intercept send
- * request events to Azure storage and allow concurrent OOB I/Os.
- */
-@InterfaceAudience.Private
-public final class SendRequestIntercept extends StorageEvent {
-
- private static final String ALLOW_ALL_REQUEST_PRECONDITIONS = "*";
-
- /**
- * Hidden default constructor for SendRequestIntercept.
- */
- private SendRequestIntercept() {
- }
-
- /**
- * Binds a new lister to the operation context so the WASB file system can
- * appropriately intercept sends and allow concurrent OOB I/Os. This
- * by-passes the blob immutability check when reading streams.
- *
- * @param opContext the operation context assocated with this request.
- */
- public static void bind(OperationContext opContext) {
- opContext.getSendingRequestEventHandler().addListener(new SendRequestIntercept());
- }
-
- /**
- * Handler which processes the sending request event from Azure SDK. The
- * handler simply sets reset the conditional header to make all read requests
- * unconditional if reads with concurrent OOB writes are allowed.
- *
- * @param sendEvent
- * - send event context from Windows Azure SDK.
- */
- @Override
- public void eventOccurred(SendingRequestEvent sendEvent) {
-
- if (!(sendEvent.getConnectionObject() instanceof HttpURLConnection)) {
- // Pass if there is no HTTP connection associated with this send
- // request.
- return;
- }
-
- // Capture the HTTP URL connection object and get size of the payload for
- // the request.
- HttpURLConnection urlConnection = (HttpURLConnection) sendEvent
- .getConnectionObject();
-
- // Determine whether this is a download request by checking that the request
- // method
- // is a "GET" operation.
- if (urlConnection.getRequestMethod().equalsIgnoreCase("GET")) {
- // If concurrent reads on OOB writes are allowed, reset the if-match
- // condition on the conditional header.
- urlConnection.setRequestProperty(HeaderConstants.IF_MATCH,
- ALLOW_ALL_REQUEST_PRECONDITIONS);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ShellDecryptionKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ShellDecryptionKeyProvider.java
deleted file mode 100644
index d9d6fc3cb4812..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ShellDecryptionKeyProvider.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.Shell;
-
-/**
- * Shell decryption key provider which invokes an external script that will
- * perform the key decryption.
- */
-@InterfaceAudience.Private
-public class ShellDecryptionKeyProvider extends SimpleKeyProvider {
- static final String KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT =
- "fs.azure.shellkeyprovider.script";
-
- @Override
- public String getStorageAccountKey(String accountName, Configuration conf)
- throws KeyProviderException {
- String envelope = super.getStorageAccountKey(accountName, conf);
-
- final String command = conf.get(KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT);
- if (command == null) {
- throw new KeyProviderException(
- "Script path is not specified via fs.azure.shellkeyprovider.script");
- }
-
- String[] cmd = command.split(" ");
- String[] cmdWithEnvelope = Arrays.copyOf(cmd, cmd.length + 1);
- cmdWithEnvelope[cmdWithEnvelope.length - 1] = envelope;
-
- String decryptedKey = null;
- try {
- decryptedKey = Shell.execCommand(cmdWithEnvelope);
- } catch (IOException ex) {
- throw new KeyProviderException(ex);
- }
-
- // trim any whitespace
- return decryptedKey.trim();
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java
deleted file mode 100644
index 64811e13ee9eb..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.ProviderUtils;
-
-/**
- * Key provider that simply returns the storage account key from the
- * configuration as plaintext.
- */
-@InterfaceAudience.Private
-public class SimpleKeyProvider implements KeyProvider {
- private static final Logger LOG = LoggerFactory.getLogger(SimpleKeyProvider.class);
-
- protected static final String KEY_ACCOUNT_KEY_PREFIX =
- "fs.azure.account.key.";
-
- @Override
- public String getStorageAccountKey(String accountName, Configuration conf)
- throws KeyProviderException {
- String key = null;
- try {
- Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
- conf, NativeAzureFileSystem.class);
- char[] keyChars = c.getPassword(getStorageAccountKeyName(accountName));
- if (keyChars != null) {
- key = new String(keyChars);
- }
- } catch(IOException ioe) {
- LOG.warn("Unable to get key from credential providers.", ioe);
- }
- return key;
- }
-
- protected String getStorageAccountKeyName(String accountName) {
- return KEY_ACCOUNT_KEY_PREFIX + accountName;
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
deleted file mode 100644
index dbb38491d7f55..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
+++ /dev/null
@@ -1,793 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.EnumSet;
-import java.util.HashMap;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RetryPolicyFactory;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobListingDetails;
-import com.microsoft.azure.storage.blob.BlobProperties;
-import com.microsoft.azure.storage.blob.BlockEntry;
-import com.microsoft.azure.storage.blob.BlockListingFilter;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.blob.CopyState;
-import com.microsoft.azure.storage.blob.ListBlobItem;
-import com.microsoft.azure.storage.blob.PageRange;
-
-/**
- * This is a very thin layer over the methods exposed by the Windows Azure
- * Storage SDK that we need for WASB implementation. This base class has a real
- * implementation that just simply redirects to the SDK, and a memory-backed one
- * that's used for unit tests.
- *
- * IMPORTANT: all the methods here must remain very simple redirects since code
- * written here can't be properly unit tested.
- */
-@InterfaceAudience.Private
-abstract class StorageInterface {
-
- /**
- * Sets the timeout to use when making requests to the storage service.
- *
- * The server timeout interval begins at the time that the complete request
- * has been received by the service, and the server begins processing the
- * response. If the timeout interval elapses before the response is returned
- * to the client, the operation times out. The timeout interval resets with
- * each retry, if the request is retried.
- *
- * The default timeout interval for a request made via the service client is
- * 90 seconds. You can change this value on the service client by setting this
- * property, so that all subsequent requests made via the service client will
- * use the new timeout interval. You can also change this value for an
- * individual request, by setting the
- * {@link com.microsoft.azure.storage.RequestOptions#timeoutIntervalInMs}
- * property.
- *
- * If you are downloading a large blob, you should increase the value of the
- * timeout beyond the default value.
- *
- * @param timeoutInMs
- * The timeout, in milliseconds, to use when making requests to the
- * storage service.
- */
- public abstract void setTimeoutInMs(int timeoutInMs);
-
- /**
- * Sets the RetryPolicyFactory object to use when making service requests.
- *
- * @param retryPolicyFactory
- * the RetryPolicyFactory object to use when making service requests.
- */
- public abstract void setRetryPolicyFactory(
- final RetryPolicyFactory retryPolicyFactory);
-
- /**
- * Creates a new Blob service client.
- *
- * @param account cloud storage account.
- */
- public abstract void createBlobClient(CloudStorageAccount account);
-
- /**
- * Creates an instance of the CloudBlobClient class using the
- * specified Blob service endpoint.
- *
- * @param baseUri
- * A java.net.URI object that represents the Blob
- * service endpoint used to create the client.
- */
- public abstract void createBlobClient(URI baseUri);
-
- /**
- * Creates an instance of the CloudBlobClient class using the
- * specified Blob service endpoint and account credentials.
- *
- * @param baseUri
- * A java.net.URI object that represents the Blob
- * service endpoint used to create the client.
- * @param credentials
- * A {@link StorageCredentials} object that represents the account
- * credentials.
- */
- public abstract void createBlobClient(URI baseUri,
- StorageCredentials credentials);
-
- /**
- * Returns the credentials for the Blob service, as configured for the storage
- * account.
- *
- * @return A {@link StorageCredentials} object that represents the credentials
- * for this storage account.
- */
- public abstract StorageCredentials getCredentials();
-
- /**
- * Returns a reference to a {@link CloudBlobContainerWrapper} object that
- * represents the cloud blob container for the specified address.
- *
- * @param name
- * A String that represents the name of the container.
- * @return A {@link CloudBlobContainerWrapper} object that represents a
- * reference to the cloud blob container.
- *
- * @throws URISyntaxException
- * If the resource URI is invalid.
- * @throws StorageException
- * If a storage service error occurred.
- */
- public abstract CloudBlobContainerWrapper getContainerReference(String name)
- throws URISyntaxException, StorageException;
-
- /**
- * A thin wrapper over the
- * {@link com.microsoft.azure.storage.blob.CloudBlobDirectory} class
- * that simply redirects calls to the real object except in unit tests.
- */
- @InterfaceAudience.Private
- public abstract static class CloudBlobDirectoryWrapper implements
- ListBlobItem {
- /**
- * Returns the URI for this directory.
- *
- * @return A java.net.URI object that represents the URI for
- * this directory.
- */
- public abstract URI getUri();
-
- /**
- * Returns an enumerable collection of blob items whose names begin with the
- * specified prefix, using the specified flat or hierarchical option,
- * listing details options, request options, and operation context.
- *
- * @param prefix
- * A String that represents the prefix of the blob
- * name.
- * @param useFlatBlobListing
- * true to indicate that the returned list will be
- * flat; false to indicate that the returned list will
- * be hierarchical.
- * @param listingDetails
- * A java.util.EnumSet object that contains
- * {@link BlobListingDetails} values that indicate whether
- * snapshots, metadata, and/or uncommitted blocks are returned.
- * Committed blocks are always returned.
- * @param options
- * A {@link BlobRequestOptions} object that specifies any
- * additional options for the request. Specifying null
- * will use the default request options from the associated service
- * client ({@link com.microsoft.azure.storage.blob.CloudBlobClient}).
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @return An enumerable collection of {@link ListBlobItem} objects that
- * represent the block items whose names begin with the specified
- * prefix in this directory.
- *
- * @throws StorageException
- * If a storage service error occurred.
- * @throws URISyntaxException
- * If the resource URI is invalid.
- */
- public abstract Iterable listBlobs(String prefix,
- boolean useFlatBlobListing, EnumSet listingDetails,
- BlobRequestOptions options, OperationContext opContext)
- throws URISyntaxException, StorageException;
- }
-
- /**
- * A thin wrapper over the
- * {@link com.microsoft.azure.storage.blob.CloudBlobContainer} class
- * that simply redirects calls to the real object except in unit tests.
- */
- @InterfaceAudience.Private
- public abstract static class CloudBlobContainerWrapper {
- /**
- * Returns the name of the container.
- *
- * @return A String that represents the name of the container.
- */
- public abstract String getName();
-
- /**
- * Returns a value that indicates whether the container exists, using the
- * specified operation context.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @return true if the container exists, otherwise
- * false.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- public abstract boolean exists(OperationContext opContext)
- throws StorageException;
-
- /**
- * Returns the metadata for the container.
- *
- * @return A java.util.HashMap object that represents the
- * metadata for the container.
- */
- public abstract HashMap getMetadata();
-
- /**
- * Sets the metadata for the container.
- *
- * @param metadata
- * A java.util.HashMap object that represents the
- * metadata being assigned to the container.
- */
- public abstract void setMetadata(HashMap metadata);
-
- /**
- * Downloads the container's attributes, which consist of metadata and
- * properties, using the specified operation context.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- public abstract void downloadAttributes(OperationContext opContext)
- throws StorageException;
-
- /**
- * Uploads the container's metadata using the specified operation context.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- public abstract void uploadMetadata(OperationContext opContext)
- throws StorageException;
-
- /**
- * Creates the container using the specified operation context.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- public abstract void create(OperationContext opContext)
- throws StorageException;
-
- /**
- * Returns a wrapper for a CloudBlobDirectory.
- *
- * @param relativePath
- * A String that represents the name of the directory,
- * relative to the container
- *
- * @throws StorageException
- * If a storage service error occurred.
- *
- * @throws URISyntaxException
- * If URI syntax exception occurred.
- */
- public abstract CloudBlobDirectoryWrapper getDirectoryReference(
- String relativePath) throws URISyntaxException, StorageException;
-
- /**
- * Returns a wrapper for a CloudBlockBlob.
- *
- * @param relativePath
- * A String that represents the name of the blob,
- * relative to the container
- *
- * @throws StorageException
- * If a storage service error occurred.
- *
- * @throws URISyntaxException
- * If URI syntax exception occurred.
- */
- public abstract CloudBlobWrapper getBlockBlobReference(
- String relativePath) throws URISyntaxException, StorageException;
-
- /**
- * Returns a wrapper for a CloudPageBlob.
- *
- * @param relativePath
- * A String that represents the name of the blob, relative to the container
- *
- * @throws StorageException
- * If a storage service error occurred.
- *
- * @throws URISyntaxException
- * If URI syntax exception occurred.
- */
- public abstract CloudBlobWrapper getPageBlobReference(String relativePath)
- throws URISyntaxException, StorageException;
- }
-
-
- /**
- * A thin wrapper over the {@link CloudBlob} class that simply redirects calls
- * to the real object except in unit tests.
- */
- @InterfaceAudience.Private
- public interface CloudBlobWrapper extends ListBlobItem {
- /**
- * Returns the URI for this blob.
- *
- * @return A java.net.URI object that represents the URI for
- * the blob.
- */
- URI getUri();
-
- /**
- * Returns the metadata for the blob.
- *
- * @return A java.util.HashMap object that represents the
- * metadata for the blob.
- */
- HashMap getMetadata();
-
- /**
- * Sets the metadata for the blob.
- *
- * @param metadata
- * A java.util.HashMap object that contains the
- * metadata being assigned to the blob.
- */
- void setMetadata(HashMap metadata);
-
- /**
- * Copies an existing blob's contents, properties, and metadata to this instance of the CloudBlob
- * class, using the specified operation context.
- *
- * @param sourceBlob
- * A CloudBlob object that represents the source blob to copy.
- * @param options
- * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
- * null will use the default request options from the associated service client (
- * {@link CloudBlobClient}).
- * @param opContext
- * An {@link OperationContext} object that represents the context for the current operation. This object
- * is used to track requests to the storage service, and to provide additional runtime information about
- * the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- * @throws URISyntaxException
- *
- */
- public abstract void startCopyFromBlob(CloudBlobWrapper sourceBlob,
- BlobRequestOptions options, OperationContext opContext, boolean overwriteDestination)
- throws StorageException, URISyntaxException;
-
- /**
- * Returns the blob's copy state.
- *
- * @return A {@link CopyState} object that represents the copy state of the
- * blob.
- */
- CopyState getCopyState();
-
- /**
- * Downloads a range of bytes from the blob to the given byte buffer, using the specified request options and
- * operation context.
- *
- * @param offset
- * The byte offset to use as the starting point for the source.
- * @param length
- * The number of bytes to read.
- * @param buffer
- * The byte buffer, as an array of bytes, to which the blob bytes are downloaded.
- * @param bufferOffset
- * The byte offset to use as the starting point for the target.
- * @param options
- * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
- * null will use the default request options from the associated service client (
- * {@link CloudBlobClient}).
- * @param opContext
- * An {@link OperationContext} object that represents the context for the current operation. This object
- * is used to track requests to the storage service, and to provide additional runtime information about
- * the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- void downloadRange(final long offset, final long length,
- final OutputStream outStream, final BlobRequestOptions options,
- final OperationContext opContext)
- throws StorageException, IOException;
-
- /**
- * Deletes the blob using the specified operation context.
- *
- * A blob that has snapshots cannot be deleted unless the snapshots are also
- * deleted. If a blob has snapshots, use the
- * {@link DeleteSnapshotsOption#DELETE_SNAPSHOTS_ONLY} or
- * {@link DeleteSnapshotsOption#INCLUDE_SNAPSHOTS} value in the
- * deleteSnapshotsOption parameter to specify how the snapshots
- * should be handled when the blob is deleted.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- void delete(OperationContext opContext, SelfRenewingLease lease)
- throws StorageException;
-
- /**
- * Checks to see if the blob exists, using the specified operation context.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @return true if the blob exists, otherwise
- * false.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- boolean exists(OperationContext opContext)
- throws StorageException;
-
- /**
- * Populates a blob's properties and metadata using the specified operation
- * context.
- *
- * This method populates the blob's system properties and user-defined
- * metadata. Before reading a blob's properties or metadata, call this
- * method or its overload to retrieve the latest values for the blob's
- * properties and metadata from the Windows Azure storage service.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- void downloadAttributes(OperationContext opContext)
- throws StorageException;
-
- /**
- * Returns the blob's properties.
- *
- * @return A {@link BlobProperties} object that represents the properties of
- * the blob.
- */
- BlobProperties getProperties();
-
- /**
- * Opens a blob input stream to download the blob using the specified
- * operation context.
- *
- * Use {@link CloudBlobClient#setStreamMinimumReadSizeInBytes} to configure
- * the read size.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @return An InputStream object that represents the stream to
- * use for reading from the blob.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- InputStream openInputStream(BlobRequestOptions options,
- OperationContext opContext) throws StorageException;
-
- /**
- * Uploads the blob's metadata to the storage service using the specified
- * lease ID, request options, and operation context.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- void uploadMetadata(OperationContext opContext)
- throws StorageException;
-
- /**
- * Uploads the blob's metadata to the storage service using the specified
- * lease ID, request options, and operation context.
- *
- * @param accessCondition
- * A {@link AccessCondition} object that represents the access conditions for the blob.
- *
- * @param options
- * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
- * null will use the default request options from the associated service client (
- * {@link CloudBlobClient}).
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context
- * for the current operation. This object is used to track requests
- * to the storage service, and to provide additional runtime
- * information about the operation.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- void uploadMetadata(AccessCondition accessCondition, BlobRequestOptions options,
- OperationContext opContext) throws StorageException;
-
- void uploadProperties(OperationContext opContext,
- SelfRenewingLease lease)
- throws StorageException;
-
- SelfRenewingLease acquireLease() throws StorageException;
-
- /**
- * Gets the minimum read block size to use with this Blob.
- *
- * @return The minimum block size, in bytes, for reading from a block blob.
- */
- int getStreamMinimumReadSizeInBytes();
-
- /**
- * Sets the minimum read block size to use with this Blob.
- *
- * @param minimumReadSizeBytes
- * The maximum block size, in bytes, for reading from a block blob
- * while using a {@link BlobInputStream} object, ranging from 512
- * bytes to 64 MB, inclusive.
- */
- void setStreamMinimumReadSizeInBytes(
- int minimumReadSizeBytes);
-
- /**
- * Sets the write block size to use with this Blob.
- *
- * @param writeBlockSizeBytes
- * The maximum block size, in bytes, for writing to a block blob
- * while using a {@link BlobOutputStream} object, ranging from 1 MB
- * to 4 MB, inclusive.
- *
- * @throws IllegalArgumentException
- * If writeBlockSizeInBytes is less than 1 MB or
- * greater than 4 MB.
- */
- void setWriteBlockSizeInBytes(int writeBlockSizeBytes);
-
- CloudBlob getBlob();
- }
-
- /**
- * A thin wrapper over the
- * {@link com.microsoft.azure.storage.blob.CloudBlockBlob} class
- * that simply redirects calls to the real object except in unit tests.
- */
- public abstract interface CloudBlockBlobWrapper
- extends CloudBlobWrapper {
- /**
- * Creates and opens an output stream to write data to the block blob using the specified
- * operation context.
- *
- * @param opContext
- * An {@link OperationContext} object that represents the context for the current operation. This object
- * is used to track requests to the storage service, and to provide additional runtime information about
- * the operation.
- *
- * @return A {@link BlobOutputStream} object used to write data to the blob.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- OutputStream openOutputStream(
- BlobRequestOptions options,
- OperationContext opContext) throws StorageException;
-
- /**
- *
- * @param filter A {@link BlockListingFilter} value that specifies whether to download
- * committed blocks, uncommitted blocks, or all blocks.
- * @param options A {@link BlobRequestOptions} object that specifies any additional options for
- * the request. Specifying null will use the default request options from
- * the associated service client ( CloudBlobClient).
- * @param opContext An {@link OperationContext} object that represents the context for the current
- * operation. This object is used to track requests to the storage service,
- * and to provide additional runtime information about the operation.
- * @return An ArrayList object of {@link BlockEntry} objects that represent the list
- * block items downloaded from the block blob.
- * @throws IOException If an I/O error occurred.
- * @throws StorageException If a storage service error occurred.
- */
- List downloadBlockList(BlockListingFilter filter, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException;
-
- /**
- *
- * @param blockId A String that represents the Base-64 encoded block ID. Note for a given blob
- * the length of all Block IDs must be identical.
- * @param accessCondition An {@link AccessCondition} object that represents the access conditions for the blob.
- * @param sourceStream An {@link InputStream} object that represents the input stream to write to the
- * block blob.
- * @param length A long which represents the length, in bytes, of the stream data,
- * or -1 if unknown.
- * @param options A {@link BlobRequestOptions} object that specifies any additional options for the
- * request. Specifying null will use the default request options from the
- * associated service client ( CloudBlobClient).
- * @param opContext An {@link OperationContext} object that represents the context for the current operation.
- * This object is used to track requests to the storage service, and to provide
- * additional runtime information about the operation.
- * @throws IOException If an I/O error occurred.
- * @throws StorageException If a storage service error occurred.
- */
- void uploadBlock(String blockId, AccessCondition accessCondition, InputStream sourceStream,
- long length, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException;
-
- /**
- *
- * @param blockList An enumerable collection of {@link BlockEntry} objects that represents the list
- * block items being committed. The size field is ignored.
- * @param accessCondition An {@link AccessCondition} object that represents the access conditions for the blob.
- * @param options A {@link BlobRequestOptions} object that specifies any additional options for the
- * request. Specifying null will use the default request options from the associated
- * service client ( CloudBlobClient).
- * @param opContext An {@link OperationContext} object that represents the context for the current operation.
- * This object is used to track requests to the storage service, and to provide additional
- * runtime information about the operation.
- * @throws IOException If an I/O error occurred.
- * @throws StorageException If a storage service error occurred.
- */
- void commitBlockList(List blockList, AccessCondition accessCondition, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException;
-
- }
-
- /**
- * A thin wrapper over the
- * {@link com.microsoft.azure.storage.blob.CloudPageBlob}
- * class that simply redirects calls to the real object except in unit tests.
- */
- public abstract interface CloudPageBlobWrapper
- extends CloudBlobWrapper {
- /**
- * Creates a page blob using the specified request options and operation context.
- *
- * @param length
- * The size, in bytes, of the page blob.
- * @param options
- * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
- * null will use the default request options from the associated service client (
- * {@link CloudBlobClient}).
- * @param opContext
- * An {@link OperationContext} object that represents the context for the current operation. This object
- * is used to track requests to the storage service, and to provide additional runtime information about
- * the operation.
- *
- * @throws IllegalArgumentException
- * If the length is not a multiple of 512.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
- void create(final long length, BlobRequestOptions options,
- OperationContext opContext) throws StorageException;
-
-
- /**
- * Uploads a range of contiguous pages, up to 4 MB in size, at the specified offset in the page blob, using the
- * specified lease ID, request options, and operation context.
- *
- * @param sourceStream
- * An InputStream object that represents the input stream to write to the page blob.
- * @param offset
- * The offset, in number of bytes, at which to begin writing the data. This value must be a multiple of
- * 512.
- * @param length
- * The length, in bytes, of the data to write. This value must be a multiple of 512.
- * @param options
- * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
- * null will use the default request options from the associated service client (
- * {@link CloudBlobClient}).
- * @param opContext
- * An {@link OperationContext} object that represents the context for the current operation. This object
- * is used to track requests to the storage service, and to provide additional runtime information about
- * the operation.
- *
- * @throws IllegalArgumentException
- * If the offset or length are not multiples of 512, or if the length is greater than 4 MB.
- * @throws IOException
- * If an I/O exception occurred.
- * @throws StorageException
- * If a storage service error occurred.
- */
- void uploadPages(final InputStream sourceStream, final long offset,
- final long length, BlobRequestOptions options,
- OperationContext opContext) throws StorageException, IOException;
-
- /**
- * Returns a collection of page ranges and their starting and ending byte offsets using the specified request
- * options and operation context.
- *
- * @param options
- * A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
- * null will use the default request options from the associated service client (
- * {@link CloudBlobClient}).
- * @param opContext
- * An {@link OperationContext} object that represents the context for the current operation. This object
- * is used to track requests to the storage service, and to provide additional runtime information about
- * the operation.
- *
- * @return An ArrayList object that represents the set of page ranges and their starting and ending
- * byte offsets.
- *
- * @throws StorageException
- * If a storage service error occurred.
- */
-
- ArrayList downloadPageRanges(BlobRequestOptions options,
- OperationContext opContext) throws StorageException;
-
- void uploadMetadata(OperationContext opContext)
- throws StorageException;
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
deleted file mode 100644
index e600f9e59da3f..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
+++ /dev/null
@@ -1,522 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import com.microsoft.azure.storage.AccessCondition;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RetryPolicyFactory;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.StorageUri;
-import com.microsoft.azure.storage.blob.BlobListingDetails;
-import com.microsoft.azure.storage.blob.BlobProperties;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.BlockEntry;
-import com.microsoft.azure.storage.blob.BlockListingFilter;
-import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import com.microsoft.azure.storage.blob.CloudPageBlob;
-import com.microsoft.azure.storage.blob.CopyState;
-import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
-import com.microsoft.azure.storage.blob.ListBlobItem;
-import com.microsoft.azure.storage.blob.PageRange;
-
-/**
- * A real implementation of the Azure interaction layer that just redirects
- * calls to the Windows Azure storage SDK.
- */
-@InterfaceAudience.Private
-class StorageInterfaceImpl extends StorageInterface {
- private CloudBlobClient serviceClient;
- private RetryPolicyFactory retryPolicyFactory;
- private int timeoutIntervalInMs;
-
- private void updateRetryPolicy() {
- if (serviceClient != null && retryPolicyFactory != null) {
- serviceClient.getDefaultRequestOptions().setRetryPolicyFactory(retryPolicyFactory);
- }
- }
-
- private void updateTimeoutInMs() {
- if (serviceClient != null && timeoutIntervalInMs > 0) {
- serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs);
- }
- }
-
- @Override
- public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) {
- this.retryPolicyFactory = retryPolicyFactory;
- updateRetryPolicy();
- }
-
- @Override
- public void setTimeoutInMs(int timeoutInMs) {
- timeoutIntervalInMs = timeoutInMs;
- updateTimeoutInMs();
- }
-
- @Override
- public void createBlobClient(CloudStorageAccount account) {
- serviceClient = account.createCloudBlobClient();
- updateRetryPolicy();
- updateTimeoutInMs();
- }
-
- @Override
- public void createBlobClient(URI baseUri) {
- createBlobClient(baseUri, (StorageCredentials)null);
- }
-
- @Override
- public void createBlobClient(URI baseUri, StorageCredentials credentials) {
- serviceClient = new CloudBlobClient(baseUri, credentials);
- updateRetryPolicy();
- updateTimeoutInMs();
- }
-
- @Override
- public StorageCredentials getCredentials() {
- return serviceClient.getCredentials();
- }
-
- @Override
- public CloudBlobContainerWrapper getContainerReference(String uri)
- throws URISyntaxException, StorageException {
- return new CloudBlobContainerWrapperImpl(
- serviceClient.getContainerReference(uri));
- }
-
- //
- // WrappingIterator
- //
-
- /**
- * This iterator wraps every ListBlobItem as they come from the listBlobs()
- * calls to their proper wrapping objects.
- */
- private static class WrappingIterator implements Iterator {
- private final Iterator present;
-
- public WrappingIterator(Iterator present) {
- this.present = present;
- }
-
- public static Iterable wrap(
- final Iterable present) {
- return new Iterable() {
- @Override
- public Iterator iterator() {
- return new WrappingIterator(present.iterator());
- }
- };
- }
-
- @Override
- public boolean hasNext() {
- return present.hasNext();
- }
-
- @Override
- public ListBlobItem next() {
- ListBlobItem unwrapped = present.next();
- if (unwrapped instanceof CloudBlobDirectory) {
- return new CloudBlobDirectoryWrapperImpl((CloudBlobDirectory) unwrapped);
- } else if (unwrapped instanceof CloudBlockBlob) {
- return new CloudBlockBlobWrapperImpl((CloudBlockBlob) unwrapped);
- } else if (unwrapped instanceof CloudPageBlob) {
- return new CloudPageBlobWrapperImpl((CloudPageBlob) unwrapped);
- } else {
- return unwrapped;
- }
- }
-
- @Override
- public void remove() {
- present.remove();
- }
- }
-
- //
- // CloudBlobDirectoryWrapperImpl
- //
- @InterfaceAudience.Private
- static class CloudBlobDirectoryWrapperImpl extends CloudBlobDirectoryWrapper {
- private final CloudBlobDirectory directory;
-
- public CloudBlobDirectoryWrapperImpl(CloudBlobDirectory directory) {
- this.directory = directory;
- }
-
- @Override
- public URI getUri() {
- return directory.getUri();
- }
-
- @Override
- public Iterable listBlobs(String prefix,
- boolean useFlatBlobListing, EnumSet listingDetails,
- BlobRequestOptions options, OperationContext opContext)
- throws URISyntaxException, StorageException {
- return WrappingIterator.wrap(directory.listBlobs(prefix,
- useFlatBlobListing, listingDetails, options, opContext));
- }
-
- @Override
- public CloudBlobContainer getContainer() throws URISyntaxException,
- StorageException {
- return directory.getContainer();
- }
-
- @Override
- public CloudBlobDirectory getParent() throws URISyntaxException,
- StorageException {
- return directory.getParent();
- }
-
- @Override
- public StorageUri getStorageUri() {
- return directory.getStorageUri();
- }
-
- }
-
- //
- // CloudBlobContainerWrapperImpl
- //
- @InterfaceAudience.Private
- static class CloudBlobContainerWrapperImpl extends CloudBlobContainerWrapper {
- private final CloudBlobContainer container;
-
- public CloudBlobContainerWrapperImpl(CloudBlobContainer container) {
- this.container = container;
- }
-
- @Override
- public String getName() {
- return container.getName();
- }
-
- @Override
- public boolean exists(OperationContext opContext) throws StorageException {
- return container.exists(AccessCondition.generateEmptyCondition(), null,
- opContext);
- }
-
- @Override
- public void create(OperationContext opContext) throws StorageException {
- container.create(null, opContext);
- }
-
- @Override
- public HashMap getMetadata() {
- return container.getMetadata();
- }
-
- @Override
- public void setMetadata(HashMap metadata) {
- container.setMetadata(metadata);
- }
-
- @Override
- public void downloadAttributes(OperationContext opContext)
- throws StorageException {
- container.downloadAttributes(AccessCondition.generateEmptyCondition(),
- null, opContext);
- }
-
- @Override
- public void uploadMetadata(OperationContext opContext)
- throws StorageException {
- container.uploadMetadata(AccessCondition.generateEmptyCondition(), null,
- opContext);
- }
-
- @Override
- public CloudBlobDirectoryWrapper getDirectoryReference(String relativePath)
- throws URISyntaxException, StorageException {
-
- CloudBlobDirectory dir = container.getDirectoryReference(relativePath);
- return new CloudBlobDirectoryWrapperImpl(dir);
- }
-
- @Override
- public CloudBlobWrapper getBlockBlobReference(String relativePath)
- throws URISyntaxException, StorageException {
-
- return new CloudBlockBlobWrapperImpl(container.getBlockBlobReference(relativePath));
- }
-
- @Override
- public CloudBlobWrapper getPageBlobReference(String relativePath)
- throws URISyntaxException, StorageException {
- return new CloudPageBlobWrapperImpl(
- container.getPageBlobReference(relativePath));
- }
-
- }
-
- abstract static class CloudBlobWrapperImpl implements CloudBlobWrapper {
- private final CloudBlob blob;
-
- @Override
- public CloudBlob getBlob() {
- return blob;
- }
-
- public URI getUri() {
- return getBlob().getUri();
- }
-
- protected CloudBlobWrapperImpl(CloudBlob blob) {
- this.blob = blob;
- }
-
- @Override
- public HashMap getMetadata() {
- return getBlob().getMetadata();
- }
-
- @Override
- public void delete(OperationContext opContext, SelfRenewingLease lease)
- throws StorageException {
- getBlob().delete(DeleteSnapshotsOption.NONE, getLeaseCondition(lease),
- null, opContext);
- }
-
- /**
- * Return and access condition for this lease, or else null if
- * there's no lease.
- */
- private AccessCondition getLeaseCondition(SelfRenewingLease lease) {
- AccessCondition leaseCondition = null;
- if (lease != null) {
- leaseCondition = AccessCondition.generateLeaseCondition(lease.getLeaseID());
- }
- return leaseCondition;
- }
-
- @Override
- public boolean exists(OperationContext opContext)
- throws StorageException {
- return getBlob().exists(null, null, opContext);
- }
-
- @Override
- public void downloadAttributes(
- OperationContext opContext) throws StorageException {
- getBlob().downloadAttributes(null, null, opContext);
- }
-
- @Override
- public BlobProperties getProperties() {
- return getBlob().getProperties();
- }
-
- @Override
- public void setMetadata(HashMap metadata) {
- getBlob().setMetadata(metadata);
- }
-
- @Override
- public InputStream openInputStream(
- BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- return getBlob().openInputStream(null, options, opContext);
- }
-
- public OutputStream openOutputStream(
- BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- return ((CloudBlockBlob) getBlob()).openOutputStream(null, options, opContext);
- }
-
- public void upload(InputStream sourceStream, OperationContext opContext)
- throws StorageException, IOException {
- getBlob().upload(sourceStream, 0, null, null, opContext);
- }
-
- @Override
- public CloudBlobContainer getContainer() throws URISyntaxException,
- StorageException {
- return getBlob().getContainer();
- }
-
- @Override
- public CloudBlobDirectory getParent() throws URISyntaxException,
- StorageException {
- return getBlob().getParent();
- }
-
- @Override
- public void uploadMetadata(OperationContext opContext)
- throws StorageException {
- uploadMetadata(null, null, opContext);
- }
-
- @Override
- public void uploadMetadata(AccessCondition accessConditions, BlobRequestOptions options,
- OperationContext opContext) throws StorageException{
- getBlob().uploadMetadata(accessConditions, options, opContext);
- }
-
- public void uploadProperties(OperationContext opContext, SelfRenewingLease lease)
- throws StorageException {
-
- // Include lease in request if lease not null.
- getBlob().uploadProperties(getLeaseCondition(lease), null, opContext);
- }
-
- @Override
- public int getStreamMinimumReadSizeInBytes() {
- return getBlob().getStreamMinimumReadSizeInBytes();
- }
-
- @Override
- public void setStreamMinimumReadSizeInBytes(int minimumReadSizeBytes) {
- getBlob().setStreamMinimumReadSizeInBytes(minimumReadSizeBytes);
- }
-
- @Override
- public void setWriteBlockSizeInBytes(int writeBlockSizeBytes) {
- getBlob().setStreamWriteSizeInBytes(writeBlockSizeBytes);
- }
-
- @Override
- public StorageUri getStorageUri() {
- return getBlob().getStorageUri();
- }
-
- @Override
- public CopyState getCopyState() {
- return getBlob().getCopyState();
- }
-
- @Override
- public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
- OperationContext opContext, boolean overwriteDestination)
- throws StorageException, URISyntaxException {
- AccessCondition dstAccessCondition =
- overwriteDestination
- ? null
- : AccessCondition.generateIfNotExistsCondition();
- getBlob().startCopy(sourceBlob.getBlob().getQualifiedUri(),
- null, dstAccessCondition, options, opContext);
- }
-
- @Override
- public void downloadRange(long offset, long length, OutputStream outStream,
- BlobRequestOptions options, OperationContext opContext)
- throws StorageException, IOException {
-
- getBlob().downloadRange(offset, length, outStream, null, options, opContext);
- }
-
- @Override
- public SelfRenewingLease acquireLease() throws StorageException {
- return new SelfRenewingLease(this, false);
- }
- }
-
-
- //
- // CloudBlockBlobWrapperImpl
- //
-
- static class CloudBlockBlobWrapperImpl extends CloudBlobWrapperImpl implements CloudBlockBlobWrapper {
- public CloudBlockBlobWrapperImpl(CloudBlockBlob blob) {
- super(blob);
- }
-
- public OutputStream openOutputStream(
- BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- return ((CloudBlockBlob) getBlob()).openOutputStream(null, options, opContext);
- }
-
- public void upload(InputStream sourceStream, OperationContext opContext)
- throws StorageException, IOException {
- getBlob().upload(sourceStream, 0, null, null, opContext);
- }
-
- public void uploadProperties(OperationContext opContext)
- throws StorageException {
- getBlob().uploadProperties(null, null, opContext);
- }
-
- @Override
- public List downloadBlockList(BlockListingFilter filter, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException {
- return ((CloudBlockBlob) getBlob()).downloadBlockList(filter, null, options, opContext);
-
- }
-
- @Override
- public void uploadBlock(String blockId, AccessCondition accessCondition, InputStream sourceStream,
- long length, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException {
- ((CloudBlockBlob) getBlob()).uploadBlock(blockId, sourceStream, length, accessCondition, options, opContext);
- }
-
- @Override
- public void commitBlockList(List blockList, AccessCondition accessCondition, BlobRequestOptions options,
- OperationContext opContext) throws IOException, StorageException {
- ((CloudBlockBlob) getBlob()).commitBlockList(blockList, accessCondition, options, opContext);
- }
- }
-
- static class CloudPageBlobWrapperImpl extends CloudBlobWrapperImpl implements CloudPageBlobWrapper {
- public CloudPageBlobWrapperImpl(CloudPageBlob blob) {
- super(blob);
- }
-
- public void create(final long length, BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- ((CloudPageBlob) getBlob()).create(length, null, options, opContext);
- }
-
- public void uploadPages(final InputStream sourceStream, final long offset,
- final long length, BlobRequestOptions options, OperationContext opContext)
- throws StorageException, IOException {
- ((CloudPageBlob) getBlob()).uploadPages(sourceStream, offset, length, null,
- options, opContext);
- }
-
- public ArrayList downloadPageRanges(BlobRequestOptions options,
- OperationContext opContext) throws StorageException {
- return ((CloudPageBlob) getBlob()).downloadPageRanges(
- null, options, opContext);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
deleted file mode 100644
index f8aed2612a857..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SyncableDataOutputStream.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.StreamCapabilities;
-import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.impl.StoreImplementationUtils;
-
-/**
- * Support the Syncable interface on top of a DataOutputStream.
- * This allows passing the sync/hflush/hsync calls through to the
- * wrapped stream passed in to the constructor. This is required
- * for HBase when wrapping a PageBlobOutputStream used as a write-ahead log.
- */
-public class SyncableDataOutputStream extends DataOutputStream
- implements Syncable, StreamCapabilities {
-
- private static final Logger LOG = LoggerFactory.getLogger(SyncableDataOutputStream.class);
-
- public SyncableDataOutputStream(OutputStream out) {
- super(out);
- }
-
- /**
- * Get a reference to the wrapped output stream.
- *
- * @return the underlying output stream
- */
- @InterfaceAudience.LimitedPrivate({"HDFS"})
- public OutputStream getOutStream() {
- return out;
- }
-
- @Override
- public boolean hasCapability(String capability) {
- return StoreImplementationUtils.hasCapability(out, capability);
- }
-
- @Override
- public void hflush() throws IOException {
- if (out instanceof Syncable) {
- ((Syncable) out).hflush();
- }
- }
-
- @Override
- public void hsync() throws IOException {
- if (out instanceof Syncable) {
- ((Syncable) out).hsync();
- }
- }
-
- @Override
- public void close() throws IOException {
- IOException ioeFromFlush = null;
- try {
- flush();
- } catch (IOException e) {
- ioeFromFlush = e;
- throw e;
- } finally {
- try {
- this.out.close();
- } catch (IOException e) {
- // If there was an Exception during flush(), the Azure SDK will throw back the
- // same when we call close on the same stream. When try and finally both throw
- // Exception, Java will use Throwable#addSuppressed for one of the Exception so
- // that the caller will get one exception back. When within this, if both
- // Exceptions are equal, it will throw back IllegalStateException. This makes us
- // to throw back a non IOE. The below special handling is to avoid this.
- if (ioeFromFlush == e) {
- // Do nothing..
- // The close() call gave back the same IOE which flush() gave. Just swallow it
- LOG.debug("flush() and close() throwing back same Exception. Just swallowing the latter", e);
- } else {
- // Let Java handle 2 different Exceptions been thrown from try and finally.
- throw e;
- }
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java
deleted file mode 100644
index dd354d78e77f3..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasb.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-
-/**
- * WASB implementation of AbstractFileSystem.
- * This impl delegates to the old FileSystem
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Wasb extends DelegateToFileSystem {
-
- Wasb(final URI theUri, final Configuration conf) throws IOException,
- URISyntaxException {
- super(theUri, new NativeAzureFileSystem(), conf, "wasb", false);
- }
-
- @Override
- public int getUriDefaultPort() {
- return -1;
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
deleted file mode 100644
index eff9248dffe42..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * Exception that gets thrown during the authorization failures
- * in WASB.
- */
-public class WasbAuthorizationException extends AzureException {
-
- private static final long serialVersionUID = 1L;
-
- public WasbAuthorizationException(String message) {
- super(message);
- }
-
- public WasbAuthorizationException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public WasbAuthorizationException(Throwable t) {
- super(t);
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
deleted file mode 100644
index 7c63d4b8e3c39..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * Different authorization operations supported
- * in WASB.
- */
-
-public enum WasbAuthorizationOperations {
-
- READ, WRITE, EXECUTE;
-
- @Override
- public String toString() {
- switch(this) {
- case READ:
- return "read";
- case WRITE:
- return "write";
- default:
- throw new IllegalArgumentException(
- "Invalid Authorization Operation");
- }
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
deleted file mode 100644
index af0e95483302d..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Interface to implement authorization support in WASB.
- * API's of this interface will be implemented in the
- * StorageInterface Layer before making calls to Azure
- * Storage.
- */
-public interface WasbAuthorizerInterface {
- /**
- * Initializer method
- * @param conf - Configuration object
- * @throws WasbAuthorizationException - On authorization exceptions
- * @throws IOException - When not able to reach the authorizer
- */
- public void init(Configuration conf)
- throws WasbAuthorizationException, IOException;
-
- /**
- * Authorizer API to authorize access in WASB.
-
- * @param wasbAbolutePath : Absolute WASB Path used for access.
- * @param accessType : Type of access
- * @param owner : owner of the file/folder specified in the wasb path
- * @return : true - If access allowed false - If access is not allowed.
- * @throws WasbAuthorizationException - On authorization exceptions
- * @throws IOException - When not able to reach the authorizer
- */
- boolean authorize(String wasbAbolutePath, String accessType, String owner)
- throws WasbAuthorizationException, IOException;
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java
deleted file mode 100644
index 578586166f40d..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbFsck.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-
-/**
- * An fsck tool implementation for WASB that does various admin/cleanup/recovery
- * tasks on the WASB file system.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class WasbFsck extends Configured implements Tool {
- private FileSystem mockFileSystemForTesting = null;
- private static final String LOST_AND_FOUND_PATH = "/lost+found";
- private boolean pathNameWarning = false;
-
- public WasbFsck(Configuration conf) {
- super(conf);
- }
-
- /**
- * For testing purposes, set the file system to use here instead of relying on
- * getting it from the FileSystem class based on the URI.
- *
- * @param fileSystem
- * The file system to use.
- */
- @VisibleForTesting
- public void setMockFileSystemForTesting(FileSystem fileSystem) {
- this.mockFileSystemForTesting = fileSystem;
- }
-
- @Override
- public int run(String[] args) throws Exception {
- if (doPrintUsage(Arrays.asList(args))) {
- printUsage();
- return -1;
- }
- Path pathToCheck = null;
- boolean doRecover = false;
- boolean doDelete = false;
- for (String arg : args) {
- if (!arg.startsWith("-")) {
- if (pathToCheck != null) {
- System.err
- .println("Can't specify multiple paths to check on the command-line");
- return 1;
- }
- pathToCheck = new Path(arg);
- } else if (arg.equals("-move")) {
- doRecover = true;
- } else if (arg.equals("-delete")) {
- doDelete = true;
- }
- }
- if (doRecover && doDelete) {
- System.err
- .println("Conflicting options: can't specify both -move and -delete.");
- return 1;
- }
- if (pathToCheck == null) {
- pathToCheck = new Path("/"); // Check everything.
- }
- FileSystem fs;
- if (mockFileSystemForTesting == null) {
- fs = FileSystem.get(pathToCheck.toUri(), getConf());
- } else {
- fs = mockFileSystemForTesting;
- }
-
- if (!recursiveCheckChildPathName(fs, fs.makeQualified(pathToCheck))) {
- pathNameWarning = true;
- }
-
- if (!(fs instanceof NativeAzureFileSystem)) {
- System.err
- .println("Can only check WASB file system. Instead I'm asked to"
- + " check: " + fs.getUri());
- return 2;
- }
- NativeAzureFileSystem wasbFs = (NativeAzureFileSystem) fs;
- if (doRecover) {
- System.out.println("Recovering files with dangling data under: "
- + pathToCheck);
- wasbFs.recoverFilesWithDanglingTempData(pathToCheck, new Path(
- LOST_AND_FOUND_PATH));
- } else if (doDelete) {
- System.out.println("Deleting temp files with dangling data under: "
- + pathToCheck);
- wasbFs.deleteFilesWithDanglingTempData(pathToCheck);
- } else {
- System.out.println("Please specify -move or -delete");
- }
- return 0;
- }
-
- public boolean getPathNameWarning() {
- return pathNameWarning;
- }
-
- /**
- * Recursively check if a given path and its child paths have colons in their
- * names. It returns true if none of them has a colon or this path does not
- * exist, and false otherwise.
- */
- private boolean recursiveCheckChildPathName(FileSystem fs, Path p)
- throws IOException {
- if (p == null) {
- return true;
- }
- FileStatus status;
- try {
- status = fs.getFileStatus(p);
- } catch (FileNotFoundException e) {
- System.out.println("Path " + p + " does not exist!");
- return true;
- }
-
- if (status.isFile()) {
- if (containsColon(p)) {
- System.out.println("Warning: file " + p + " has a colon in its name.");
- return false;
- } else {
- return true;
- }
- } else {
- boolean flag;
- if (containsColon(p)) {
- System.out.println("Warning: directory " + p
- + " has a colon in its name.");
- flag = false;
- } else {
- flag = true;
- }
- FileStatus[] listed = fs.listStatus(p);
- for (FileStatus l : listed) {
- if (!recursiveCheckChildPathName(fs, l.getPath())) {
- flag = false;
- }
- }
- return flag;
- }
- }
-
- private boolean containsColon(Path p) {
- return p.toUri().getPath().toString().contains(":");
- }
-
- private static void printUsage() {
- System.out.println("Usage: WasbFSck [] [-move | -delete]");
- System.out.println("\t\tstart checking from this path");
- System.out.println("\t-move\tmove any files whose upload was interrupted"
- + " mid-stream to " + LOST_AND_FOUND_PATH);
- System.out
- .println("\t-delete\tdelete any files whose upload was interrupted"
- + " mid-stream");
- ToolRunner.printGenericCommandUsage(System.out);
- }
-
- private boolean doPrintUsage(List args) {
- return args.contains("-H");
- }
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new WasbFsck(new Configuration()), args);
- System.exit(res);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallException.java
deleted file mode 100644
index 43c1b618362b0..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * Exception that gets thrown when a remote call
- * made from WASB to external cred service fails.
- */
-
-public class WasbRemoteCallException extends AzureException {
-
- private static final long serialVersionUID = 1L;
-
- public WasbRemoteCallException(String message) {
- super(message);
- }
-
- public WasbRemoteCallException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public WasbRemoteCallException(Throwable t) {
- super(t);
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
deleted file mode 100644
index e6e0c9379ef37..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.fs.azure.security.Constants;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.NameValuePair;
-import org.apache.http.StatusLine;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.InterruptedIOException;
-import java.net.InetAddress;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Random;
-
-/**
- * Helper class the has constants and helper methods
- * used in WASB when integrating with a remote http cred
- * service. Currently, remote service will be used to generate
- * SAS keys.
- */
-public class WasbRemoteCallHelper {
-
- public static final Logger LOG =
- LoggerFactory.getLogger(WasbRemoteCallHelper.class);
- /**
- * Return code when the remote call is successful. {@value}
- */
- public static final int REMOTE_CALL_SUCCESS_CODE = 0;
-
- /**
- * Application Json content type.
- */
- private static final String APPLICATION_JSON = "application/json";
-
- /**
- * Max content length of the response.
- */
- private static final int MAX_CONTENT_LENGTH = 1024;
-
- /**
- * Client instance to be used for making the remote call.
- */
- private HttpClient client = null;
-
- private Random random = new Random();
-
- private RetryPolicy retryPolicy = null;
-
- public WasbRemoteCallHelper(RetryPolicy retryPolicy) {
- this.client = HttpClientBuilder.create().build();
- this.retryPolicy = retryPolicy;
- }
-
- @VisibleForTesting public void updateHttpClient(HttpClient client) {
- this.client = client;
- }
-
- /**
- * Helper method to make remote HTTP Get request.
- *
- * @param urls - Service urls to be used, if one fails try another.
- * @param path - URL endpoint for the resource.
- * @param queryParams - list of query parameters
- * @param httpMethod - http Method to be used.
- * @return Http Response body returned as a string. The caller
- * is expected to semantically understand the response.
- * @throws IOException when there an error in executing the remote http request.
- */
- public String makeRemoteRequest(String[] urls, String path,
- List queryParams, String httpMethod) throws IOException {
-
- return retryableRequest(urls, path, queryParams, httpMethod);
- }
-
- protected String retryableRequest(String[] urls, String path,
- List queryParams, String httpMethod) throws IOException {
- HttpResponse response = null;
- HttpUriRequest httpRequest = null;
-
- /**
- * Get the index of local url if any. If list of urls contains strings like
- * "https://localhost:" or "http://localhost", consider it as local url and
- * give it affinity more than other urls in the list.
- */
-
- int indexOfLocalUrl = -1;
- for (int i = 0; i < urls.length; i++) {
- if (urls[i].toLowerCase().startsWith("https://localhost:") || urls[i]
- .toLowerCase().startsWith("http://localhost:")) {
- indexOfLocalUrl = i;
- }
- }
-
- boolean requiresNewAuth = false;
- for (int retry = 0, index = (indexOfLocalUrl != -1)
- ? indexOfLocalUrl
- : random
- .nextInt(urls.length);; retry++, index++) {
- if (index >= urls.length) {
- index = index % urls.length;
- }
- /**
- * If the first request fails to localhost, then randomly pick the next url
- * from the remaining urls in the list, so that load can be balanced.
- */
- if (indexOfLocalUrl != -1 && retry == 1) {
- index = (index + random.nextInt(urls.length)) % urls.length;
- if (index == indexOfLocalUrl) {
- index = (index + 1) % urls.length;
- }
- }
- try {
- httpRequest =
- getHttpRequest(urls, path, queryParams, index, httpMethod,
- requiresNewAuth);
- httpRequest.setHeader("Accept", APPLICATION_JSON);
- response = client.execute(httpRequest);
- StatusLine statusLine = response.getStatusLine();
- if (statusLine == null
- || statusLine.getStatusCode() != HttpStatus.SC_OK) {
- requiresNewAuth =
- (statusLine == null)
- || (statusLine.getStatusCode() == HttpStatus.SC_UNAUTHORIZED);
-
- throw new WasbRemoteCallException(
- httpRequest.getURI().toString() + ":" + ((statusLine != null)
- ? statusLine.toString()
- : "NULL"));
- } else {
- requiresNewAuth = false;
- }
-
- Header contentTypeHeader = response.getFirstHeader("Content-Type");
- if (contentTypeHeader == null || !APPLICATION_JSON
- .equals(contentTypeHeader.getValue())) {
- throw new WasbRemoteCallException(
- httpRequest.getURI().toString() + ":"
- + "Content-Type mismatch: expected: " + APPLICATION_JSON
- + ", got " + ((contentTypeHeader != null) ? contentTypeHeader
- .getValue() : "NULL"));
- }
-
- Header contentLengthHeader = response.getFirstHeader("Content-Length");
- if (contentLengthHeader == null) {
- throw new WasbRemoteCallException(
- httpRequest.getURI().toString() + ":"
- + "Content-Length header missing");
- }
-
- try {
- if (Integer.parseInt(contentLengthHeader.getValue())
- > MAX_CONTENT_LENGTH) {
- throw new WasbRemoteCallException(
- httpRequest.getURI().toString() + ":" + "Content-Length:"
- + contentLengthHeader.getValue() + "exceeded max:"
- + MAX_CONTENT_LENGTH);
- }
- } catch (NumberFormatException nfe) {
- throw new WasbRemoteCallException(
- httpRequest.getURI().toString() + ":"
- + "Invalid Content-Length value :" + contentLengthHeader
- .getValue());
- }
-
- BufferedReader rd = null;
- StringBuilder responseBody = new StringBuilder();
- try {
- rd = new BufferedReader(
- new InputStreamReader(response.getEntity().getContent(),
- StandardCharsets.UTF_8));
- String responseLine = "";
- while ((responseLine = rd.readLine()) != null) {
- responseBody.append(responseLine);
- }
- } finally {
- rd.close();
- }
- return responseBody.toString();
- } catch (URISyntaxException uriSyntaxEx) {
- throw new WasbRemoteCallException("Encountered URISyntaxException "
- + "while building the HttpGetRequest to remote service",
- uriSyntaxEx);
- } catch (IOException e) {
- LOG.debug(e.getMessage(), e);
- try {
- shouldRetry(e, retry, (httpRequest != null)
- ? httpRequest.getURI().toString()
- : urls[index]);
- } catch (IOException ioex) {
- String message =
- "Encountered error while making remote call to " + String
- .join(",", urls) + " retried " + retry + " time(s).";
- LOG.error(message, ioex);
- throw new WasbRemoteCallException(message, ioex);
- }
- }
- }
- }
-
- protected HttpUriRequest getHttpRequest(String[] urls, String path,
- List queryParams, int urlIndex, String httpMethod,
- boolean requiresNewAuth) throws URISyntaxException, IOException {
- URIBuilder uriBuilder = null;
- uriBuilder =
- new URIBuilder(urls[urlIndex]).setPath(path).setParameters(queryParams);
- if (uriBuilder.getHost().equals("localhost")) {
- uriBuilder.setHost(InetAddress.getLocalHost().getCanonicalHostName());
- }
- HttpUriRequest httpUriRequest = null;
- switch (httpMethod) {
- case HttpPut.METHOD_NAME:
- httpUriRequest = new HttpPut(uriBuilder.build());
- break;
- case HttpPost.METHOD_NAME:
- httpUriRequest = new HttpPost(uriBuilder.build());
- break;
- default:
- httpUriRequest = new HttpGet(uriBuilder.build());
- break;
- }
- return httpUriRequest;
- }
-
- private void shouldRetry(final IOException ioe, final int retry,
- final String url) throws IOException {
- CharSequence authenticationExceptionMessage =
- Constants.AUTHENTICATION_FAILED_ERROR_MESSAGE;
- if (ioe instanceof WasbRemoteCallException && ioe.getMessage()
- .equals(authenticationExceptionMessage)) {
- throw ioe;
- }
- try {
- final RetryPolicy.RetryAction a = (retryPolicy != null)
- ? retryPolicy
- .shouldRetry(ioe, retry, 0, true)
- : RetryPolicy.RetryAction.FAIL;
-
- boolean isRetry = a.action == RetryPolicy.RetryAction.RetryDecision.RETRY;
- boolean isFailoverAndRetry =
- a.action == RetryPolicy.RetryAction.RetryDecision.FAILOVER_AND_RETRY;
-
- if (isRetry || isFailoverAndRetry) {
- LOG.debug("Retrying connect to Remote service:{}. Already tried {}"
- + " time(s); retry policy is {}, " + "delay {}ms.", url, retry,
- retryPolicy, a.delayMillis);
-
- Thread.sleep(a.delayMillis);
- return;
- }
- } catch (InterruptedIOException e) {
- LOG.warn(e.getMessage(), e);
- Thread.currentThread().interrupt();
- return;
- } catch (Exception e) {
- LOG.warn("Original exception is ", ioe);
- throw new WasbRemoteCallException(e.getMessage(), e);
- }
- LOG.debug("Not retrying anymore, already retried the urls {} time(s)",
- retry);
- throw new WasbRemoteCallException(
- url + ":" + "Encountered IOException while making remote call", ioe);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java
deleted file mode 100644
index 0b4a7824b58c9..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-
-/**
- * WASB implementation of AbstractFileSystem for wasbs scheme.
- * This impl delegates to the old FileSystem
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class Wasbs extends DelegateToFileSystem {
-
- Wasbs(final URI theUri, final Configuration conf) throws IOException,
- URISyntaxException {
- super(theUri, new NativeAzureFileSystem(), conf, "wasbs", false);
- }
-
- @Override
- public int getUriDefaultPort() {
- return -1;
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemInstrumentation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemInstrumentation.java
deleted file mode 100644
index 6cce271b227d0..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemInstrumentation.java
+++ /dev/null
@@ -1,396 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.metrics;
-
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metrics2.MetricsCollector;
-import org.apache.hadoop.metrics2.MetricsInfo;
-import org.apache.hadoop.metrics2.MetricsSource;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
-
-/**
- * A metrics source for the WASB file system to track all the metrics we care
- * about for getting a clear picture of the performance/reliability/interaction
- * of the Hadoop cluster with Azure Storage.
- */
-@Metrics(about="Metrics for WASB", context="azureFileSystem")
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public final class AzureFileSystemInstrumentation implements MetricsSource {
-
- public static final String METRIC_TAG_FILESYSTEM_ID = "wasbFileSystemId";
- public static final String METRIC_TAG_ACCOUNT_NAME = "accountName";
- public static final String METRIC_TAG_CONTAINTER_NAME = "containerName";
-
- public static final String WASB_WEB_RESPONSES = "wasb_web_responses";
- public static final String WASB_BYTES_WRITTEN =
- "wasb_bytes_written_last_second";
- public static final String WASB_BYTES_READ =
- "wasb_bytes_read_last_second";
- public static final String WASB_RAW_BYTES_UPLOADED =
- "wasb_raw_bytes_uploaded";
- public static final String WASB_RAW_BYTES_DOWNLOADED =
- "wasb_raw_bytes_downloaded";
- public static final String WASB_FILES_CREATED = "wasb_files_created";
- public static final String WASB_FILES_DELETED = "wasb_files_deleted";
- public static final String WASB_DIRECTORIES_CREATED = "wasb_directories_created";
- public static final String WASB_DIRECTORIES_DELETED = "wasb_directories_deleted";
- public static final String WASB_UPLOAD_RATE =
- "wasb_maximum_upload_bytes_per_second";
- public static final String WASB_DOWNLOAD_RATE =
- "wasb_maximum_download_bytes_per_second";
- public static final String WASB_UPLOAD_LATENCY =
- "wasb_average_block_upload_latency_ms";
- public static final String WASB_DOWNLOAD_LATENCY =
- "wasb_average_block_download_latency_ms";
- public static final String WASB_CLIENT_ERRORS = "wasb_client_errors";
- public static final String WASB_SERVER_ERRORS = "wasb_server_errors";
-
- /**
- * Config key for how big the rolling window size for latency metrics should
- * be (in seconds).
- */
- private static final String KEY_ROLLING_WINDOW_SIZE = "fs.azure.metrics.rolling.window.size";
-
- private final MetricsRegistry registry =
- new MetricsRegistry("azureFileSystem")
- .setContext("azureFileSystem");
- private final MutableCounterLong numberOfWebResponses =
- registry.newCounter(
- WASB_WEB_RESPONSES,
- "Total number of web responses obtained from Azure Storage",
- 0L);
- private AtomicLong inMemoryNumberOfWebResponses = new AtomicLong(0);
- private final MutableCounterLong numberOfFilesCreated =
- registry.newCounter(
- WASB_FILES_CREATED,
- "Total number of files created through the WASB file system.",
- 0L);
- private final MutableCounterLong numberOfFilesDeleted =
- registry.newCounter(
- WASB_FILES_DELETED,
- "Total number of files deleted through the WASB file system.",
- 0L);
- private final MutableCounterLong numberOfDirectoriesCreated =
- registry.newCounter(
- WASB_DIRECTORIES_CREATED,
- "Total number of directories created through the WASB file system.",
- 0L);
- private final MutableCounterLong numberOfDirectoriesDeleted =
- registry.newCounter(
- WASB_DIRECTORIES_DELETED,
- "Total number of directories deleted through the WASB file system.",
- 0L);
- private final MutableGaugeLong bytesWrittenInLastSecond =
- registry.newGauge(
- WASB_BYTES_WRITTEN,
- "Total number of bytes written to Azure Storage during the last second.",
- 0L);
- private final MutableGaugeLong bytesReadInLastSecond =
- registry.newGauge(
- WASB_BYTES_READ,
- "Total number of bytes read from Azure Storage during the last second.",
- 0L);
- private final MutableGaugeLong maximumUploadBytesPerSecond =
- registry.newGauge(
- WASB_UPLOAD_RATE,
- "The maximum upload rate encountered to Azure Storage in bytes/second.",
- 0L);
- private final MutableGaugeLong maximumDownloadBytesPerSecond =
- registry.newGauge(
- WASB_DOWNLOAD_RATE,
- "The maximum download rate encountered to Azure Storage in bytes/second.",
- 0L);
- private final MutableCounterLong rawBytesUploaded =
- registry.newCounter(
- WASB_RAW_BYTES_UPLOADED,
- "Total number of raw bytes (including overhead) uploaded to Azure"
- + " Storage.",
- 0L);
- private final MutableCounterLong rawBytesDownloaded =
- registry.newCounter(
- WASB_RAW_BYTES_DOWNLOADED,
- "Total number of raw bytes (including overhead) downloaded from Azure"
- + " Storage.",
- 0L);
- private final MutableCounterLong clientErrors =
- registry.newCounter(
- WASB_CLIENT_ERRORS,
- "Total number of client-side errors by WASB (excluding 404).",
- 0L);
- private final MutableCounterLong serverErrors =
- registry.newCounter(
- WASB_SERVER_ERRORS,
- "Total number of server-caused errors by WASB.",
- 0L);
- private final MutableGaugeLong averageBlockUploadLatencyMs;
- private final MutableGaugeLong averageBlockDownloadLatencyMs;
- private long currentMaximumUploadBytesPerSecond;
- private long currentMaximumDownloadBytesPerSecond;
- private static final int DEFAULT_LATENCY_ROLLING_AVERAGE_WINDOW =
- 5; // seconds
- private final RollingWindowAverage currentBlockUploadLatency;
- private final RollingWindowAverage currentBlockDownloadLatency;
- private UUID fileSystemInstanceId;
-
- public AzureFileSystemInstrumentation(Configuration conf) {
- fileSystemInstanceId = UUID.randomUUID();
- registry.tag("wasbFileSystemId",
- "A unique identifier for the file ",
- fileSystemInstanceId.toString());
- final int rollingWindowSizeInSeconds =
- conf.getInt(KEY_ROLLING_WINDOW_SIZE,
- DEFAULT_LATENCY_ROLLING_AVERAGE_WINDOW);
- averageBlockUploadLatencyMs =
- registry.newGauge(
- WASB_UPLOAD_LATENCY,
- String.format("The average latency in milliseconds of uploading a single block"
- + ". The average latency is calculated over a %d-second rolling"
- + " window.", rollingWindowSizeInSeconds),
- 0L);
- averageBlockDownloadLatencyMs =
- registry.newGauge(
- WASB_DOWNLOAD_LATENCY,
- String.format("The average latency in milliseconds of downloading a single block"
- + ". The average latency is calculated over a %d-second rolling"
- + " window.", rollingWindowSizeInSeconds),
- 0L);
- currentBlockUploadLatency =
- new RollingWindowAverage(rollingWindowSizeInSeconds * 1000);
- currentBlockDownloadLatency =
- new RollingWindowAverage(rollingWindowSizeInSeconds * 1000);
- }
-
- /**
- * The unique identifier for this file system in the metrics.
- * @return The unique identifier.
- */
- public UUID getFileSystemInstanceId() {
- return fileSystemInstanceId;
- }
-
- /**
- * Get the metrics registry information.
- * @return The metrics registry information.
- */
- public MetricsInfo getMetricsRegistryInfo() {
- return registry.info();
- }
-
- /**
- * Sets the account name to tag all the metrics with.
- * @param accountName The account name.
- */
- public void setAccountName(String accountName) {
- registry.tag("accountName",
- "Name of the Azure Storage account that these metrics are going against",
- accountName);
- }
-
- /**
- * Sets the container name to tag all the metrics with.
- * @param containerName The container name.
- */
- public void setContainerName(String containerName) {
- registry.tag("containerName",
- "Name of the Azure Storage container that these metrics are going against",
- containerName);
- }
-
- /**
- * Indicate that we just got a web response from Azure Storage. This should
- * be called for every web request/response we do (to get accurate metrics
- * of how we're hitting the storage service).
- */
- public void webResponse() {
- numberOfWebResponses.incr();
- inMemoryNumberOfWebResponses.incrementAndGet();
- }
-
- /**
- * Gets the current number of web responses obtained from Azure Storage.
- * @return The number of web responses.
- */
- public long getCurrentWebResponses() {
- return inMemoryNumberOfWebResponses.get();
- }
-
- /**
- * Indicate that we just created a file through WASB.
- */
- public void fileCreated() {
- numberOfFilesCreated.incr();
- }
-
- /**
- * Indicate that we just deleted a file through WASB.
- */
- public void fileDeleted() {
- numberOfFilesDeleted.incr();
- }
-
- /**
- * Indicate that we just created a directory through WASB.
- */
- public void directoryCreated() {
- numberOfDirectoriesCreated.incr();
- }
-
- /**
- * Indicate that we just deleted a directory through WASB.
- */
- public void directoryDeleted() {
- numberOfDirectoriesDeleted.incr();
- }
-
- /**
- * Sets the current gauge value for how many bytes were written in the last
- * second.
- * @param currentBytesWritten The number of bytes.
- */
- public void updateBytesWrittenInLastSecond(long currentBytesWritten) {
- bytesWrittenInLastSecond.set(currentBytesWritten);
- }
-
- /**
- * Sets the current gauge value for how many bytes were read in the last
- * second.
- * @param currentBytesRead The number of bytes.
- */
- public void updateBytesReadInLastSecond(long currentBytesRead) {
- bytesReadInLastSecond.set(currentBytesRead);
- }
-
- /**
- * Record the current bytes-per-second upload rate seen.
- * @param bytesPerSecond The bytes per second.
- */
- public synchronized void currentUploadBytesPerSecond(long bytesPerSecond) {
- if (bytesPerSecond > currentMaximumUploadBytesPerSecond) {
- currentMaximumUploadBytesPerSecond = bytesPerSecond;
- maximumUploadBytesPerSecond.set(bytesPerSecond);
- }
- }
-
- /**
- * Record the current bytes-per-second download rate seen.
- * @param bytesPerSecond The bytes per second.
- */
- public synchronized void currentDownloadBytesPerSecond(long bytesPerSecond) {
- if (bytesPerSecond > currentMaximumDownloadBytesPerSecond) {
- currentMaximumDownloadBytesPerSecond = bytesPerSecond;
- maximumDownloadBytesPerSecond.set(bytesPerSecond);
- }
- }
-
- /**
- * Indicate that we just uploaded some data to Azure storage.
- * @param numberOfBytes The raw number of bytes uploaded (including overhead).
- */
- public void rawBytesUploaded(long numberOfBytes) {
- rawBytesUploaded.incr(numberOfBytes);
- }
-
- /**
- * Indicate that we just downloaded some data to Azure storage.
- * @param numberOfBytes The raw number of bytes downloaded (including overhead).
- */
- public void rawBytesDownloaded(long numberOfBytes) {
- rawBytesDownloaded.incr(numberOfBytes);
- }
-
- /**
- * Indicate that we just uploaded a block and record its latency.
- * @param latency The latency in milliseconds.
- */
- public void blockUploaded(long latency) {
- currentBlockUploadLatency.addPoint(latency);
- }
-
- /**
- * Indicate that we just downloaded a block and record its latency.
- * @param latency The latency in milliseconds.
- */
- public void blockDownloaded(long latency) {
- currentBlockDownloadLatency.addPoint(latency);
- }
-
- /**
- * Indicate that we just encountered a client-side error.
- */
- public void clientErrorEncountered() {
- clientErrors.incr();
- }
-
- /**
- * Indicate that we just encountered a server-caused error.
- */
- public void serverErrorEncountered() {
- serverErrors.incr();
- }
-
- /**
- * Get the current rolling average of the upload latency.
- * @return rolling average of upload latency in milliseconds.
- */
- public long getBlockUploadLatency() {
- return currentBlockUploadLatency.getCurrentAverage();
- }
-
- /**
- * Get the current rolling average of the download latency.
- * @return rolling average of download latency in milliseconds.
- */
- public long getBlockDownloadLatency() {
- return currentBlockDownloadLatency.getCurrentAverage();
- }
-
- /**
- * Get the current maximum upload bandwidth.
- * @return maximum upload bandwidth in bytes per second.
- */
- public long getCurrentMaximumUploadBandwidth() {
- return currentMaximumUploadBytesPerSecond;
- }
-
- /**
- * Get the current maximum download bandwidth.
- * @return maximum download bandwidth in bytes per second.
- */
- public long getCurrentMaximumDownloadBandwidth() {
- return currentMaximumDownloadBytesPerSecond;
- }
-
- @Override
- public void getMetrics(MetricsCollector builder, boolean all) {
- averageBlockDownloadLatencyMs.set(
- currentBlockDownloadLatency.getCurrentAverage());
- averageBlockUploadLatencyMs.set(
- currentBlockUploadLatency.getCurrentAverage());
- registry.snapshot(builder.addRecord(registry.info().name()), true);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
deleted file mode 100644
index 322795ab82712..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/AzureFileSystemMetricsSystem.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.metrics2.MetricsSource;
-import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
-
-/**
- * AzureFileSystemMetricsSystem
- */
-@InterfaceAudience.Private
-public final class AzureFileSystemMetricsSystem {
- private static MetricsSystemImpl instance;
- private static int numFileSystems;
-
- //private ctor
- private AzureFileSystemMetricsSystem(){
-
- }
-
- public static synchronized void fileSystemStarted() {
- if (numFileSystems == 0) {
- instance = new MetricsSystemImpl();
- instance.init("azure-file-system");
- }
- numFileSystems++;
- }
-
- public static synchronized void fileSystemClosed() {
- if (numFileSystems == 1) {
- instance.publishMetricsNow();
- instance.stop();
- instance.shutdown();
- instance = null;
- }
- numFileSystems--;
- }
-
- public static void registerSource(String name, String desc,
- MetricsSource source) {
- //caller has to use unique name to register source
- instance.register(name, desc, source);
- }
-
- public static synchronized void unregisterSource(String name) {
- if (instance != null) {
- //publish metrics before unregister a metrics source
- instance.publishMetricsNow();
- instance.unregisterSource(name);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/BandwidthGaugeUpdater.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/BandwidthGaugeUpdater.java
deleted file mode 100644
index d0a1bd0e7fb63..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/BandwidthGaugeUpdater.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.metrics;
-
-import java.util.ArrayList;
-import java.util.Date;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
-
-/**
- * Internal implementation class to help calculate the current bytes
- * uploaded/downloaded and the maximum bandwidth gauges.
- */
-@InterfaceAudience.Private
-public final class BandwidthGaugeUpdater {
-
- public static final String THREAD_NAME = "AzureNativeFilesystemStore-UploadBandwidthUpdater";
-
- private static final int DEFAULT_WINDOW_SIZE_MS = 1000;
- private static final int PROCESS_QUEUE_INITIAL_CAPACITY = 1000;
- private int windowSizeMs;
- private ArrayList allBlocksWritten =
- createNewToProcessQueue();
- private ArrayList allBlocksRead =
- createNewToProcessQueue();
- private final Object blocksWrittenLock = new Object();
- private final Object blocksReadLock = new Object();
- private final AzureFileSystemInstrumentation instrumentation;
- private Thread uploadBandwidthUpdater;
- private volatile boolean suppressAutoUpdate = false;
-
- /**
- * Create a new updater object with default values.
- * @param instrumentation The metrics source to update.
- */
- public BandwidthGaugeUpdater(AzureFileSystemInstrumentation instrumentation) {
- this(instrumentation, DEFAULT_WINDOW_SIZE_MS, false);
- }
-
- /**
- * Create a new updater object with some overrides (used in unit tests).
- * @param instrumentation The metrics source to update.
- * @param windowSizeMs The window size to use for calculating bandwidth
- * (in milliseconds).
- * @param manualUpdateTrigger If true, then this object won't create the
- * auto-update threads, and will wait for manual
- * calls to triggerUpdate to occur.
- */
- public BandwidthGaugeUpdater(AzureFileSystemInstrumentation instrumentation,
- int windowSizeMs, boolean manualUpdateTrigger) {
- this.windowSizeMs = windowSizeMs;
- this.instrumentation = instrumentation;
- if (!manualUpdateTrigger) {
- uploadBandwidthUpdater = new SubjectInheritingThread(new UploadBandwidthUpdater(), THREAD_NAME);
- uploadBandwidthUpdater.setDaemon(true);
- uploadBandwidthUpdater.start();
- }
- }
-
- /**
- * Indicate that a block has been uploaded.
- * @param startDate The exact time the upload started.
- * @param endDate The exact time the upload ended.
- * @param length The number of bytes uploaded in the block.
- */
- public void blockUploaded(Date startDate, Date endDate, long length) {
- synchronized (blocksWrittenLock) {
- allBlocksWritten.add(new BlockTransferWindow(startDate, endDate, length));
- }
- }
-
- /**
- * Indicate that a block has been downloaded.
- * @param startDate The exact time the download started.
- * @param endDate The exact time the download ended.
- * @param length The number of bytes downloaded in the block.
- */
- public void blockDownloaded(Date startDate, Date endDate, long length) {
- synchronized (blocksReadLock) {
- allBlocksRead.add(new BlockTransferWindow(startDate, endDate, length));
- }
- }
-
- /**
- * Creates a new ArrayList to hold incoming block transfer notifications
- * before they're processed.
- * @return The newly created ArrayList.
- */
- private static ArrayList createNewToProcessQueue() {
- return new ArrayList(PROCESS_QUEUE_INITIAL_CAPACITY);
- }
-
- /**
- * Update the metrics source gauge for how many bytes were transferred
- * during the last time window.
- * @param updateWrite If true, update the write (upload) counter.
- * Otherwise update the read (download) counter.
- * @param bytes The number of bytes transferred.
- */
- private void updateBytesTransferred(boolean updateWrite, long bytes) {
- if (updateWrite) {
- instrumentation.updateBytesWrittenInLastSecond(bytes);
- }
- else {
- instrumentation.updateBytesReadInLastSecond(bytes);
- }
- }
-
- /**
- * Update the metrics source gauge for what the current transfer rate
- * is.
- * @param updateWrite If true, update the write (upload) counter.
- * Otherwise update the read (download) counter.
- * @param bytesPerSecond The number of bytes per second we're seeing.
- */
- private void updateBytesTransferRate(boolean updateWrite, long bytesPerSecond) {
- if (updateWrite) {
- instrumentation.currentUploadBytesPerSecond(bytesPerSecond);
- }
- else {
- instrumentation.currentDownloadBytesPerSecond(bytesPerSecond);
- }
- }
-
- /**
- * For unit test purposes, suppresses auto-update of the metrics
- * from the dedicated thread.
- */
- public void suppressAutoUpdate() {
- suppressAutoUpdate = true;
- }
-
- /**
- * Resumes auto-update (undo suppressAutoUpdate).
- */
- public void resumeAutoUpdate() {
- suppressAutoUpdate = false;
- }
-
- /**
- * Triggers the update of the metrics gauge based on all the blocks
- * uploaded/downloaded so far. This is typically done periodically in a
- * dedicated update thread, but exposing as public for unit test purposes.
- *
- * @param updateWrite If true, we'll update the write (upload) metrics.
- * Otherwise we'll update the read (download) ones.
- */
- public void triggerUpdate(boolean updateWrite) {
- ArrayList toProcess = null;
- synchronized (updateWrite ? blocksWrittenLock : blocksReadLock) {
- if (updateWrite && !allBlocksWritten.isEmpty()) {
- toProcess = allBlocksWritten;
- allBlocksWritten = createNewToProcessQueue();
- } else if (!updateWrite && !allBlocksRead.isEmpty()) {
- toProcess = allBlocksRead;
- allBlocksRead = createNewToProcessQueue();
- }
- }
-
- // Check to see if we have any blocks to process.
- if (toProcess == null) {
- // Nothing to process, set the current bytes and rate to zero.
- updateBytesTransferred(updateWrite, 0);
- updateBytesTransferRate(updateWrite, 0);
- return;
- }
-
- // The cut-off time for when we want to calculate rates is one
- // window size ago from now.
- long cutoffTime = new Date().getTime() - windowSizeMs;
-
- // Go through all the blocks we're processing, and calculate the
- // total number of bytes processed as well as the maximum transfer
- // rate we experienced for any single block during our time window.
- long maxSingleBlockTransferRate = 0;
- long bytesInLastSecond = 0;
- for (BlockTransferWindow currentWindow : toProcess) {
- long windowDuration = currentWindow.getEndDate().getTime()
- - currentWindow.getStartDate().getTime();
- if (windowDuration == 0) {
- // Edge case, assume it took 1 ms but we were too fast
- windowDuration = 1;
- }
- if (currentWindow.getStartDate().getTime() > cutoffTime) {
- // This block was transferred fully within our time window,
- // just add its bytes to the total.
- bytesInLastSecond += currentWindow.bytesTransferred;
- } else if (currentWindow.getEndDate().getTime() > cutoffTime) {
- // This block started its transfer before our time window,
- // interpolate to estimate how many bytes from that block
- // were actually transferred during our time window.
- long adjustedBytes = (currentWindow.getBytesTransferred()
- * (currentWindow.getEndDate().getTime() - cutoffTime))
- / windowDuration;
- bytesInLastSecond += adjustedBytes;
- }
- // Calculate the transfer rate for this block.
- long currentBlockTransferRate =
- (currentWindow.getBytesTransferred() * 1000) / windowDuration;
- maxSingleBlockTransferRate =
- Math.max(maxSingleBlockTransferRate, currentBlockTransferRate);
- }
- updateBytesTransferred(updateWrite, bytesInLastSecond);
- // The transfer rate we saw in the last second is a tricky concept to
- // define: If we saw two blocks, one 2 MB block transferred in 0.2 seconds,
- // and one 4 MB block transferred in 0.2 seconds, then the maximum rate
- // is 20 MB/s (the 4 MB block), the average of the two blocks is 15 MB/s,
- // and the aggregate rate is 6 MB/s (total of 6 MB transferred in one
- // second). As a first cut, I'm taking the definition to be the maximum
- // of aggregate or of any single block's rate (so in the example case it's
- // 6 MB/s).
- long aggregateTransferRate = bytesInLastSecond;
- long maxObservedTransferRate =
- Math.max(aggregateTransferRate, maxSingleBlockTransferRate);
- updateBytesTransferRate(updateWrite, maxObservedTransferRate);
- }
-
- /**
- * A single block transfer.
- */
- private static final class BlockTransferWindow {
- private final Date startDate;
- private final Date endDate;
- private final long bytesTransferred;
-
- public BlockTransferWindow(Date startDate, Date endDate,
- long bytesTransferred) {
- this.startDate = startDate;
- this.endDate = endDate;
- this.bytesTransferred = bytesTransferred;
- }
-
- public Date getStartDate() { return startDate; }
- public Date getEndDate() { return endDate; }
- public long getBytesTransferred() { return bytesTransferred; }
- }
-
- /**
- * The auto-update thread.
- */
- private final class UploadBandwidthUpdater implements Runnable {
- @Override
- public void run() {
- try {
- while (true) {
- Thread.sleep(windowSizeMs);
- if (!suppressAutoUpdate) {
- triggerUpdate(true);
- triggerUpdate(false);
- }
- }
- } catch (InterruptedException e) {
- }
- }
- }
-
- public void close() {
- if (uploadBandwidthUpdater != null) {
- // Interrupt and join the updater thread in death.
- uploadBandwidthUpdater.interrupt();
- try {
- uploadBandwidthUpdater.join();
- } catch (InterruptedException e) {
- }
- uploadBandwidthUpdater = null;
- }
- }
-
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java
deleted file mode 100644
index dc23354e7b253..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.metrics;
-
-import static java.net.HttpURLConnection.HTTP_NOT_FOUND; //404
-import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; //400
-import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; //500
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RequestResult;
-import com.microsoft.azure.storage.ResponseReceivedEvent;
-import com.microsoft.azure.storage.StorageEvent;
-
-
-/**
- * An event listener to the ResponseReceived event from Azure Storage that will
- * update error metrics appropriately when it gets that event.
- */
-@InterfaceAudience.Private
-public final class ErrorMetricUpdater extends StorageEvent {
- private final AzureFileSystemInstrumentation instrumentation;
- private final OperationContext operationContext;
-
- private ErrorMetricUpdater(OperationContext operationContext,
- AzureFileSystemInstrumentation instrumentation) {
- this.instrumentation = instrumentation;
- this.operationContext = operationContext;
- }
-
- /**
- * Hooks a new listener to the given operationContext that will update the
- * error metrics for the WASB file system appropriately in response to
- * ResponseReceived events.
- *
- * @param operationContext The operationContext to hook.
- * @param instrumentation The metrics source to update.
- */
- public static void hook(
- OperationContext operationContext,
- AzureFileSystemInstrumentation instrumentation) {
- ErrorMetricUpdater listener =
- new ErrorMetricUpdater(operationContext,
- instrumentation);
- operationContext.getResponseReceivedEventHandler().addListener(listener);
- }
-
- @Override
- public void eventOccurred(ResponseReceivedEvent eventArg) {
- RequestResult currentResult = operationContext.getLastResult();
- int statusCode = currentResult.getStatusCode();
- // Check if it's a client-side error: a 4xx status
- // We exclude 404 because it happens frequently during the normal
- // course of operation (each call to exists() would generate that
- // if it's not found).
- if (statusCode >= HTTP_BAD_REQUEST && statusCode < HTTP_INTERNAL_ERROR
- && statusCode != HTTP_NOT_FOUND) {
- instrumentation.clientErrorEncountered();
- } else if (statusCode >= HTTP_INTERNAL_ERROR) {
- // It's a server error: a 5xx status. Could be an Azure Storage
- // bug or (more likely) throttling.
- instrumentation.serverErrorEncountered();
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java
deleted file mode 100644
index 4c61f6817cf75..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.metrics;
-
-import java.net.HttpURLConnection;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import com.microsoft.azure.storage.Constants.HeaderConstants;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RequestResult;
-import com.microsoft.azure.storage.ResponseReceivedEvent;
-import com.microsoft.azure.storage.StorageEvent;
-
-
-/**
- * An event listener to the ResponseReceived event from Azure Storage that will
- * update metrics appropriately when it gets that event.
- */
-@InterfaceAudience.Private
-public final class ResponseReceivedMetricUpdater extends StorageEvent {
-
- private final AzureFileSystemInstrumentation instrumentation;
- private final BandwidthGaugeUpdater blockUploadGaugeUpdater;
-
- private ResponseReceivedMetricUpdater(OperationContext operationContext,
- AzureFileSystemInstrumentation instrumentation,
- BandwidthGaugeUpdater blockUploadGaugeUpdater) {
- this.instrumentation = instrumentation;
- this.blockUploadGaugeUpdater = blockUploadGaugeUpdater;
- }
-
- /**
- * Hooks a new listener to the given operationContext that will update the
- * metrics for the WASB file system appropriately in response to
- * ResponseReceived events.
- *
- * @param operationContext The operationContext to hook.
- * @param instrumentation The metrics source to update.
- * @param blockUploadGaugeUpdater The blockUploadGaugeUpdater to use.
- */
- public static void hook(
- OperationContext operationContext,
- AzureFileSystemInstrumentation instrumentation,
- BandwidthGaugeUpdater blockUploadGaugeUpdater) {
- ResponseReceivedMetricUpdater listener =
- new ResponseReceivedMetricUpdater(operationContext,
- instrumentation, blockUploadGaugeUpdater);
- operationContext.getResponseReceivedEventHandler().addListener(listener);
- }
-
- /**
- * Get the content length of the request in the given HTTP connection.
- * @param connection The connection.
- * @return The content length, or zero if not found.
- */
- private long getRequestContentLength(HttpURLConnection connection) {
- String lengthString = connection.getRequestProperty(
- HeaderConstants.CONTENT_LENGTH);
- if (lengthString != null){
- return Long.parseLong(lengthString);
- }
- else{
- return 0;
- }
- }
-
- /**
- * Gets the content length of the response in the given HTTP connection.
- * @param connection The connection.
- * @return The content length.
- */
- private long getResponseContentLength(HttpURLConnection connection) {
- return connection.getContentLength();
- }
-
- /**
- * Handle the response-received event from Azure SDK.
- */
- @Override
- public void eventOccurred(ResponseReceivedEvent eventArg) {
- instrumentation.webResponse();
- if (!(eventArg.getConnectionObject() instanceof HttpURLConnection)) {
- // Typically this shouldn't happen, but just let it pass
- return;
- }
- HttpURLConnection connection =
- (HttpURLConnection) eventArg.getConnectionObject();
- RequestResult currentResult = eventArg.getRequestResult();
- if (currentResult == null) {
- // Again, typically shouldn't happen, but let it pass
- return;
- }
-
- long requestLatency = currentResult.getStopDate().getTime()
- - currentResult.getStartDate().getTime();
-
- if (currentResult.getStatusCode() == HttpURLConnection.HTTP_CREATED
- && connection.getRequestMethod().equalsIgnoreCase("PUT")) {
- // If it's a PUT with an HTTP_CREATED status then it's a successful
- // block upload.
- long length = getRequestContentLength(connection);
- if (length > 0) {
- blockUploadGaugeUpdater.blockUploaded(
- currentResult.getStartDate(),
- currentResult.getStopDate(),
- length);
- instrumentation.rawBytesUploaded(length);
- instrumentation.blockUploaded(requestLatency);
- }
- } else if (currentResult.getStatusCode() == HttpURLConnection.HTTP_PARTIAL
- && connection.getRequestMethod().equalsIgnoreCase("GET")) {
- // If it's a GET with an HTTP_PARTIAL status then it's a successful
- // block download.
- long length = getResponseContentLength(connection);
- if (length > 0) {
- blockUploadGaugeUpdater.blockDownloaded(
- currentResult.getStartDate(),
- currentResult.getStopDate(),
- length);
- instrumentation.rawBytesDownloaded(length);
- instrumentation.blockDownloaded(requestLatency);
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/RollingWindowAverage.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/RollingWindowAverage.java
deleted file mode 100644
index 184907a9e7bf7..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/RollingWindowAverage.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.metrics;
-
-import java.util.ArrayDeque;
-import java.util.Date;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Helper class to calculate rolling-window averages.
- * Used to calculate rolling-window metrics in AzureNativeFileSystem.
- */
-@InterfaceAudience.Private
-final class RollingWindowAverage {
- private final ArrayDeque currentPoints =
- new ArrayDeque();
- private final long windowSizeMs;
-
- /**
- * Create a new rolling-window average for the given window size.
- * @param windowSizeMs The size of the window in milliseconds.
- */
- public RollingWindowAverage(long windowSizeMs) {
- this.windowSizeMs = windowSizeMs;
- }
-
- /**
- * Add a new data point that just happened.
- * @param value The value of the data point.
- */
- public synchronized void addPoint(long value) {
- currentPoints.offer(new DataPoint(new Date(), value));
- cleanupOldPoints();
- }
-
- /**
- * Get the current average.
- * @return The current average.
- */
- public synchronized long getCurrentAverage() {
- cleanupOldPoints();
- if (currentPoints.isEmpty()) {
- return 0;
- }
- long sum = 0;
- for (DataPoint current : currentPoints) {
- sum += current.getValue();
- }
- return sum / currentPoints.size();
- }
-
- /**
- * Clean up points that don't count any more (are before our
- * rolling window) from our current queue of points.
- */
- private void cleanupOldPoints() {
- Date cutoffTime = new Date(new Date().getTime() - windowSizeMs);
- while (!currentPoints.isEmpty()
- && currentPoints.peekFirst().getEventTime().before(cutoffTime)) {
- currentPoints.removeFirst();
- }
- }
-
- /**
- * A single data point.
- */
- private static class DataPoint {
- private final Date eventTime;
- private final long value;
-
- public DataPoint(Date eventTime, long value) {
- this.eventTime = eventTime;
- this.value = value;
- }
-
- public Date getEventTime() {
- return eventTime;
- }
-
- public long getValue() {
- return value;
- }
-
-
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/package.html b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/package.html
deleted file mode 100644
index 5e8d6a84693e6..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/package.html
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-
-
-
-
-
-Infrastructure for a Metrics2 source that provides information on Windows
-Azure Filesystem for Hadoop instances.
-
-
-
-
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html
deleted file mode 100644
index de01683995d8f..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/package.html
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-
-
-
-A distributed implementation of {@link
-org.apache.hadoop.fs.FileSystem} for reading and writing files on
-Azure Block Storage.
-This implementation is blob-based and stores files on Azure in their native form for
-interoperability with other Azure tools.
-
-
-
-
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
deleted file mode 100644
index 792fe0a65b679..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-/**
- * Constants for used with WASB security implementation.
- */
-public final class Constants {
-
- /**
- * The configuration property to enable Kerberos support.
- */
-
- public static final String AZURE_KERBEROS_SUPPORT_PROPERTY_NAME =
- "fs.azure.enable.kerberos.support";
- /**
- * The configuration property to enable SPNEGO token cache.
- */
- public static final String AZURE_ENABLE_SPNEGO_TOKEN_CACHE =
- "fs.azure.enable.spnego.token.cache";
-
- /**
- * Parameter to be used for impersonation.
- */
- public static final String DOAS_PARAM = "doas";
- /**
- * Error message for Authentication failures.
- */
- public static final String AUTHENTICATION_FAILED_ERROR_MESSAGE =
- "Authentication Failed ";
-
- private Constants() {
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
deleted file mode 100644
index 9c40325e217e7..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.util.JsonSerialization;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * Utility class to parse JSON.
- */
-public final class JsonUtils {
- public static final Logger LOG = LoggerFactory.getLogger(JsonUtils.class);
-
- private JsonUtils() {
- }
-
- public static Map parse(final String jsonString) throws IOException {
- try {
- return JsonSerialization.mapReader().readValue(jsonString);
- } catch (Exception e) {
- LOG.debug("JSON Parsing exception: {} while parsing {}", e.getMessage(),
- jsonString);
- if (jsonString.toLowerCase(Locale.ENGLISH).contains("server error")) {
- LOG.error(
- "Internal Server Error was encountered while making a request");
- }
- throw new IOException("JSON Parsing Error: " + e.getMessage(), e);
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/RemoteWasbDelegationTokenManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/RemoteWasbDelegationTokenManager.java
deleted file mode 100644
index 36381dc472540..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/RemoteWasbDelegationTokenManager.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.azure.SecureWasbRemoteCallHelper;
-import org.apache.hadoop.fs.azure.WasbRemoteCallHelper;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryUtils;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.utils.URIBuilder;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Class to manage delegation token operations by making rest call to remote service.
- */
-public class RemoteWasbDelegationTokenManager
- implements WasbDelegationTokenManager {
-
- /**
- * Configuration parameter name expected in the configuration
- * object to provide the url of the delegation token service to fetch the delegation tokens.
- */
- public static final String KEY_DELEGATION_TOKEN_SERVICE_URLS =
- "fs.azure.delegation.token.service.urls";
- /**
- * Configuration key to enable http retry policy for delegation token service calls.
- */
- public static final String DT_MANAGER_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY =
- "fs.azure.delegationtokenservice.http.retry.policy.enabled";
- /**
- * Configuration key for delegation token service http retry policy spec.
- */
- public static final String DT_MANAGER_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY =
- "fs.azure.delegationtokenservice.http.retry.policy.spec";
- /**
- * Default remote delegation token manager endpoint.
- */
- private static final String DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT =
- "/tokenmanager/v1";
- /**
- * Default for delegation token service http retry policy spec.
- */
- private static final String DT_MANAGER_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
- "10,3,100,2";
-
- private static final boolean
- DT_MANAGER_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
-
- private static final Text WASB_DT_SERVICE_NAME = new Text("WASB_DT_SERVICE");
- /**
- * Query parameter value for Getting delegation token http request
- */
- private static final String GET_DELEGATION_TOKEN_OP = "GETDELEGATIONTOKEN";
- /**
- * Query parameter value for renewing delegation token http request
- */
- private static final String RENEW_DELEGATION_TOKEN_OP =
- "RENEWDELEGATIONTOKEN";
- /**
- * Query parameter value for canceling the delegation token http request
- */
- private static final String CANCEL_DELEGATION_TOKEN_OP =
- "CANCELDELEGATIONTOKEN";
- /**
- * op parameter to represent the operation.
- */
- private static final String OP_PARAM_KEY_NAME = "op";
- /**
- * renewer parameter to represent the renewer of the delegation token.
- */
- private static final String RENEWER_PARAM_KEY_NAME = "renewer";
- /**
- * service parameter to represent the service which returns delegation tokens.
- */
- private static final String SERVICE_PARAM_KEY_NAME = "service";
- /**
- * token parameter to represent the delegation token.
- */
- private static final String TOKEN_PARAM_KEY_NAME = "token";
- private WasbRemoteCallHelper remoteCallHelper;
- private String[] dtServiceUrls;
- private boolean isSpnegoTokenCacheEnabled;
-
- public RemoteWasbDelegationTokenManager(Configuration conf)
- throws IOException {
- RetryPolicy retryPolicy = RetryUtils.getMultipleLinearRandomRetry(conf,
- DT_MANAGER_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY,
- DT_MANAGER_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
- DT_MANAGER_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
- DT_MANAGER_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT);
- this.isSpnegoTokenCacheEnabled =
- conf.getBoolean(Constants.AZURE_ENABLE_SPNEGO_TOKEN_CACHE, true);
-
- remoteCallHelper = new SecureWasbRemoteCallHelper(retryPolicy, true,
- isSpnegoTokenCacheEnabled);
- this.dtServiceUrls =
- conf.getTrimmedStrings(KEY_DELEGATION_TOKEN_SERVICE_URLS);
- if (this.dtServiceUrls == null || this.dtServiceUrls.length <= 0) {
- throw new IOException(
- KEY_DELEGATION_TOKEN_SERVICE_URLS + " config not set"
- + " in configuration.");
- }
- }
-
- @Override
- public Token getDelegationToken(
- String renewer) throws IOException {
- URIBuilder uriBuilder =
- new URIBuilder().setPath(DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT)
- .addParameter(OP_PARAM_KEY_NAME, GET_DELEGATION_TOKEN_OP)
- .addParameter(RENEWER_PARAM_KEY_NAME, renewer)
- .addParameter(SERVICE_PARAM_KEY_NAME,
- WASB_DT_SERVICE_NAME.toString());
- String responseBody = remoteCallHelper
- .makeRemoteRequest(dtServiceUrls, uriBuilder.getPath(),
- uriBuilder.getQueryParams(), HttpGet.METHOD_NAME);
- return TokenUtils.toDelegationToken(JsonUtils.parse(responseBody));
- }
-
- @Override
- public long renewDelegationToken(Token token)
- throws IOException {
- URIBuilder uriBuilder =
- new URIBuilder().setPath(DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT)
- .addParameter(OP_PARAM_KEY_NAME, RENEW_DELEGATION_TOKEN_OP)
- .addParameter(TOKEN_PARAM_KEY_NAME, token.encodeToUrlString());
-
- String responseBody = remoteCallHelper
- .makeRemoteRequest(dtServiceUrls, uriBuilder.getPath(),
- uriBuilder.getQueryParams(), HttpPut.METHOD_NAME);
-
- Map parsedResp = JsonUtils.parse(responseBody);
- return ((Number) parsedResp.get("long")).longValue();
- }
-
- @Override
- public void cancelDelegationToken(Token token)
- throws IOException {
- URIBuilder uriBuilder =
- new URIBuilder().setPath(DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT)
- .addParameter(OP_PARAM_KEY_NAME, CANCEL_DELEGATION_TOKEN_OP)
- .addParameter(TOKEN_PARAM_KEY_NAME, token.encodeToUrlString());
- remoteCallHelper.makeRemoteRequest(dtServiceUrls, uriBuilder.getPath(),
- uriBuilder.getQueryParams(), HttpPut.METHOD_NAME);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/SpnegoToken.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/SpnegoToken.java
deleted file mode 100644
index fba4e4142f59a..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/SpnegoToken.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-
-/**
- * Class to represent SPNEGO token.
- */
-public class SpnegoToken {
- private AuthenticatedURL.Token token;
- private long expiryTime;
- private static final long TOKEN_VALIDITY_TIME_IN_MS = 60 * 60 * 1000L;
-
- public SpnegoToken(AuthenticatedURL.Token token) {
- this.token = token;
- //set the expiry time of the token to be 60 minutes,
- // actual token will be valid for more than few hours and treating token as opaque.
- this.expiryTime = System.currentTimeMillis() + TOKEN_VALIDITY_TIME_IN_MS;
- }
-
- public AuthenticatedURL.Token getToken() {
- return token;
- }
-
- public long getExpiryTime() {
- return expiryTime;
- }
-
- public boolean isTokenValid() {
- return (expiryTime >= System.currentTimeMillis());
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/TokenUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/TokenUtils.java
deleted file mode 100644
index 90b9082eb7cef..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/TokenUtils.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Utility methods common for token management
- */
-public final class TokenUtils {
- public static final Logger LOG = LoggerFactory.getLogger(TokenUtils.class);
- public static final String URL_STRING = "urlString";
-
- private TokenUtils() {
- }
-
- public static Token toDelegationToken(
- final Map inputMap) throws IOException {
- final Map m = (Map) inputMap.get(Token.class.getSimpleName());
- return (Token) toToken(m);
- }
-
- public static Token toToken(final Map m)
- throws IOException {
- if (m == null) {
- return null;
- }
- String urlString = (String) m.get(URL_STRING);
- if (urlString != null) {
- final Token token = new Token<>();
- LOG.debug("Read url string param - {}", urlString);
- token.decodeFromUrlString(urlString);
- return token;
- }
- return null;
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
deleted file mode 100644
index 530e04572e2b0..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
-
-/**
- * Delegation token Identifier for WASB delegation tokens.
- */
-public class WasbDelegationTokenIdentifier extends DelegationTokenIdentifier {
- public static final Text TOKEN_KIND = new Text("WASB delegation");
-
- public WasbDelegationTokenIdentifier(){
- super(TOKEN_KIND);
- }
-
- public WasbDelegationTokenIdentifier(Text kind) {
- super(kind);
- }
-
- public WasbDelegationTokenIdentifier(Text kind, Text owner, Text renewer,
- Text realUser) {
- super(kind, owner, renewer, realUser);
- }
-
- @Override
- public Text getKind() {
- return TOKEN_KIND;
- }
-
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenManager.java
deleted file mode 100644
index 1d7341600718d..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenManager.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
-
-import java.io.IOException;
-
-/**
- * Interface for Managing the Delegation tokens.
- */
-public interface WasbDelegationTokenManager {
-
- /**
- * Get Delegation token
- * @param renewer delegation token renewer
- * @return delegation token
- * @throws IOException when error in getting the delegation token
- */
- Token getDelegationToken(String renewer)
- throws IOException;
-
- /**
- * Renew the delegation token
- * @param token delegation token.
- * @return renewed time.
- * @throws IOException when error in renewing the delegation token
- */
- long renewDelegationToken(Token token) throws IOException;
-
- /**
- * Cancel the delegation token
- * @param token delegation token.
- * @throws IOException when error in cancelling the delegation token.
- */
- void cancelDelegationToken(Token token) throws IOException;
-}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
deleted file mode 100644
index 6df76475f5594..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenRenewer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Token Renewer for renewing WASB delegation tokens with remote service.
- */
-public class WasbTokenRenewer extends TokenRenewer {
- public static final Logger LOG =
- LoggerFactory.getLogger(WasbTokenRenewer.class);
-
- /**
- * Checks if this particular object handles the Kind of token passed.
- * @param kind the kind of the token
- * @return true if it handles passed token kind false otherwise.
- */
- @Override
- public boolean handleKind(Text kind) {
- return WasbDelegationTokenIdentifier.TOKEN_KIND.equals(kind);
- }
-
- /**
- * Checks if passed token is managed.
- * @param token the token being checked
- * @return true if it is managed.
- * @throws IOException thrown when evaluating if token is managed.
- */
- @Override
- public boolean isManaged(Token token) throws IOException {
- return true;
- }
-
- /**
- * Renew the delegation token.
- * @param token token to renew.
- * @param conf configuration object.
- * @return extended expiry time of the token.
- * @throws IOException thrown when trying get current user.
- * @throws InterruptedException thrown when thread is interrupted
- */
- @Override
- public long renew(final Token token, Configuration conf)
- throws IOException, InterruptedException {
- LOG.debug("Renewing the delegation token");
- return getInstance(conf).renewDelegationToken(token);
- }
-
- /**
- * Cancel the delegation token.
- * @param token token to cancel.
- * @param conf configuration object.
- * @throws IOException thrown when trying get current user.
- * @throws InterruptedException thrown when thread is interrupted.
- */
- @Override
- public void cancel(final Token token, Configuration conf)
- throws IOException, InterruptedException {
- LOG.debug("Cancelling the delegation token");
- getInstance(conf).cancelDelegationToken(token);
- }
-
- private WasbDelegationTokenManager getInstance(Configuration conf)
- throws IOException {
- return new RemoteWasbDelegationTokenManager(conf);
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package-info.java
deleted file mode 100644
index 1e1bfbe3aea2f..0000000000000
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.fs.azure.security;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index 90169185863a6..702ad8014f0c4 100644
--- a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -14,4 +14,3 @@
# limitations under the License.
org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenIdentifier
-org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
index d889534c73cd8..159e6296457e7 100644
--- a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
+++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
@@ -14,4 +14,3 @@
# limitations under the License.
org.apache.hadoop.fs.azurebfs.security.AbfsTokenRenewer
-org.apache.hadoop.fs.azure.security.WasbTokenRenewer
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/deprecated_wasb.md b/hadoop-tools/hadoop-azure/src/site/markdown/deprecated_wasb.md
deleted file mode 100644
index b020296412312..0000000000000
--- a/hadoop-tools/hadoop-azure/src/site/markdown/deprecated_wasb.md
+++ /dev/null
@@ -1,564 +0,0 @@
-
-
-# Azure Blob Storage Support by Deprecated WASB Driver
-
-
-
-See also:
-
-* [WASB Deprecation](./wasb.html)
-* [ABFS](./index.html)
-* [Namespace Disabled Accounts on ABFS](./fns_blob.html)
-* [Testing](./testing_azure.html)
-
-## Introduction
-
-The `hadoop-azure` module provides support for integration with
-[Azure Blob Storage](http://azure.microsoft.com/en-us/documentation/services/storage/).
-The built jar file, named `hadoop-azure.jar`, also declares transitive dependencies
-on the additional artifacts it requires, notably the
-[Azure Storage SDK for Java](https://github.com/Azure/azure-storage-java).
-
-To make it part of Apache Hadoop's default classpath, simply make sure that
-`HADOOP_OPTIONAL_TOOLS` in `hadoop-env.sh` has `'hadoop-azure` in the list.
-Example:
-
-```bash
-export HADOOP_OPTIONAL_TOOLS="hadoop-azure,hadoop-azure-datalake"
-```
-## Features
-
-* Read and write data stored in an Azure Blob Storage account.
-* Present a hierarchical file system view by implementing the standard [`Hadoop
- FileSystem`](../api/org/apache/hadoop/fs/FileSystem.html) interface.
-* Supports configuration of multiple Azure Blob Storage accounts.
-* Supports both block blobs (suitable for most use cases, such as MapReduce) and
- page blobs (suitable for continuous write use cases, such as an HBase
- write-ahead log).
-* Reference file system paths using URLs using the `wasb` scheme.
-* Also reference file system paths using URLs with the `wasbs` scheme for SSL
- encrypted access.
-* Can act as a source of data in a MapReduce job, or a sink.
-* Tested on both Linux and Windows.
-* Tested at scale.
-
-## Limitations
-
-* File owner and group are persisted, but the permissions model is not enforced.
- Authorization occurs at the level of the entire Azure Blob Storage account.
-* File last access time is not tracked.
-
-## Usage
-
-### Concepts
-
-The Azure Blob Storage data model presents 3 core concepts:
-
-* **Storage Account**: All access is done through a storage account.
-* **Container**: A container is a grouping of multiple blobs. A storage account
- may have multiple containers. In Hadoop, an entire file system hierarchy is
- stored in a single container. It is also possible to configure multiple
- containers, effectively presenting multiple file systems that can be referenced
- using distinct URLs.
-* **Blob**: A file of any type and size. In Hadoop, files are stored in blobs.
- The internal implementation also uses blobs to persist the file system
- hierarchy and other metadata.
-
-### Configuring Credentials
-
-Usage of Azure Blob Storage requires configuration of credentials. Typically
-this is set in core-site.xml. The configuration property name is of the form
-`fs.azure.account.key..blob.core.windows.net` and the value is the
-access key. **The access key is a secret that protects access to your storage
-account. Do not share the access key (or the core-site.xml file) with an
-untrusted party.**
-
-For example:
-
-```xml
-
- fs.azure.account.key.youraccount.blob.core.windows.net
- YOUR ACCESS KEY
-
-```
-In many Hadoop clusters, the core-site.xml file is world-readable. It is possible to
-protect the access key within a credential provider as well. This provides an encrypted
-file format along with protection with file permissions.
-
-#### Protecting the Azure Credentials for WASB with Credential Providers
-
-To protect these credentials from prying eyes, it is recommended that you use
-the credential provider framework to securely store them and access them
-through configuration. The following describes its use for Azure credentials
-in WASB FileSystem.
-
-For additional reading on the credential provider API see:
-[Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html).
-
-##### End to End Steps for Distcp and WASB with Credential Providers
-
-###### provision
-
-```bash
-% hadoop credential create fs.azure.account.key.youraccount.blob.core.windows.net -value 123
- -provider localjceks://file/home/lmccay/wasb.jceks
-```
-
-###### configure core-site.xml or command line system property
-
-```xml
-
- hadoop.security.credential.provider.path
- localjceks://file/home/lmccay/wasb.jceks
- Path to interrogate for protected credentials.
-
-```
-
-###### distcp
-
-```bash
-% hadoop distcp
- [-D hadoop.security.credential.provider.path=localjceks://file/home/lmccay/wasb.jceks]
- hdfs://hostname:9001/user/lmccay/007020615 wasb://yourcontainer@youraccount.blob.core.windows.net/testDir/
-```
-
-NOTE: You may optionally add the provider path property to the distcp command line instead of
-added job specific configuration to a generic core-site.xml. The square brackets above illustrate
-this capability.
-
-#### Protecting the Azure Credentials for WASB within an Encrypted File
-
-In addition to using the credential provider framework to protect your credentials, it's
-also possible to configure it in encrypted form. An additional configuration property
-specifies an external program to be invoked by Hadoop processes to decrypt the
-key. The encrypted key value is passed to this external program as a command
-line argument:
-
-```xml
-
- fs.azure.account.keyprovider.youraccount
- org.apache.hadoop.fs.azure.ShellDecryptionKeyProvider
-
-
-
- fs.azure.account.key.youraccount.blob.core.windows.net
- YOUR ENCRYPTED ACCESS KEY
-
-
-
- fs.azure.shellkeyprovider.script
- PATH TO DECRYPTION PROGRAM
-
-
-```
-
-### Block Blob with Compaction Support and Configuration
-
-Block blobs are the default kind of blob and are good for most big-data use
-cases. However, block blobs have strict limit of 50,000 blocks per blob.
-To prevent reaching the limit WASB, by default, does not upload new block to
-the service after every `hflush()` or `hsync()`.
-
-For most of the cases, combining data from multiple `write()` calls in
-blocks of 4Mb is a good optimization. But, in others cases, like HBase log files,
-every call to `hflush()` or `hsync()` must upload the data to the service.
-
-Block blobs with compaction upload the data to the cloud service after every
-`hflush()`/`hsync()`. To mitigate the limit of 50000 blocks, `hflush()
-`/`hsync()` runs once compaction process, if number of blocks in the blob
-is above 32,000.
-
-Block compaction search and replaces a sequence of small blocks with one big
-block. That means there is associated cost with block compaction: reading
-small blocks back to the client and writing it again as one big block.
-
-In order to have the files you create be block blobs with block compaction
-enabled, the client must set the configuration variable
-`fs.azure.block.blob.with.compaction.dir` to a comma-separated list of
-folder names.
-
-For example:
-
-```xml
-
- fs.azure.block.blob.with.compaction.dir
- /hbase/WALs,/data/myblobfiles
-
-```
-
-### Page Blob Support and Configuration
-
-The Azure Blob Storage interface for Hadoop supports two kinds of blobs,
-[block blobs and page blobs](http://msdn.microsoft.com/en-us/library/azure/ee691964.aspx).
-Block blobs are the default kind of blob and are good for most big-data use
-cases, like input data for Hive, Pig, analytical map-reduce jobs etc. Page blob
-handling in hadoop-azure was introduced to support HBase log files. Page blobs
-can be written any number of times, whereas block blobs can only be appended to
-50,000 times before you run out of blocks and your writes will fail. That won't
-work for HBase logs, so page blob support was introduced to overcome this
-limitation.
-
-Page blobs can be up to 1TB in size, larger than the maximum 200GB size for block
-blobs.
-You should stick to block blobs for most usage, and page blobs are only tested in context of HBase write-ahead logs.
-
-In order to have the files you create be page blobs, you must set the
-configuration variable `fs.azure.page.blob.dir` to a comma-separated list of
-folder names.
-
-For example:
-
-```xml
-
- fs.azure.page.blob.dir
- /hbase/WALs,/hbase/oldWALs,/data/mypageblobfiles
-
-```
-
-You can set this to simply / to make all files page blobs.
-
-The configuration option `fs.azure.page.blob.size` is the default initial
-size for a page blob. It must be 128MB or greater, and no more than 1TB,
-specified as an integer number of bytes.
-
-The configuration option `fs.azure.page.blob.extension.size` is the page blob
-extension size. This defines the amount to extend a page blob if it starts to
-get full. It must be 128MB or greater, specified as an integer number of bytes.
-
-### Custom User-Agent
-WASB passes User-Agent header to the Azure back-end. The default value
-contains WASB version, Java Runtime version, Azure Client library version, and the
-value of the configuration option `fs.azure.user.agent.prefix`. Customized User-Agent
-header enables better troubleshooting and analysis by Azure service.
-
-```xml
-
- fs.azure.user.agent.prefix
- Identifier
-
-```
-
-### Atomic Folder Rename
-
-Azure storage stores files as a flat key/value store without formal support
-for folders. The hadoop-azure file system layer simulates folders on top
-of Azure storage. By default, folder rename in the hadoop-azure file system
-layer is not atomic. That means that a failure during a folder rename
-could, for example, leave some folders in the original directory and
-some in the new one.
-
-HBase depends on atomic folder rename. Hence, a configuration setting was
-introduced called `fs.azure.atomic.rename.dir` that allows you to specify a
-comma-separated list of directories to receive special treatment so that
-folder rename is made atomic. The default value of this setting is just
-`/hbase`. Redo will be applied to finish a folder rename that fails. A file
-`-renamePending.json` may appear temporarily and is the record of
-the intention of the rename operation, to allow redo in event of a failure.
-
-For example:
-
-```xml
-
- fs.azure.atomic.rename.dir
- /hbase,/data
-
-```
-
-### Accessing wasb URLs
-
-After credentials are configured in core-site.xml, any Hadoop component may
-reference files in that Azure Blob Storage account by using URLs of the following
-format:
-
- wasb[s]://@.blob.core.windows.net/
-
-The schemes `wasb` and `wasbs` identify a URL on a file system backed by Azure
-Blob Storage. `wasb` utilizes unencrypted HTTP access for all interaction with
-the Azure Blob Storage API. `wasbs` utilizes SSL encrypted HTTPS access.
-
-For example, the following
-[FileSystem Shell](../hadoop-project-dist/hadoop-common/FileSystemShell.html)
-commands demonstrate access to a storage account named `youraccount` and a
-container named `yourcontainer`.
-
-```bash
-% hadoop fs -mkdir wasb://yourcontainer@youraccount.blob.core.windows.net/testDir
-
-% hadoop fs -put testFile wasb://yourcontainer@youraccount.blob.core.windows.net/testDir/testFile
-
-% hadoop fs -cat wasbs://yourcontainer@youraccount.blob.core.windows.net/testDir/testFile
-test file content
-```
-
-It's also possible to configure `fs.defaultFS` to use a `wasb` or `wasbs` URL.
-This causes all bare paths, such as `/testDir/testFile` to resolve automatically
-to that file system.
-
-### Append API Support and Configuration
-
-The Azure Blob Storage interface for Hadoop has optional support for Append API for
-single writer by setting the configuration `fs.azure.enable.append.support` to true.
-
-For Example:
-
-```xml
-
- fs.azure.enable.append.support
- true
-
-```
-
-It must be noted Append support in Azure Blob Storage interface DIFFERS FROM HDFS SEMANTICS. Append
-support does not enforce single writer internally but requires applications to guarantee this semantic.
-It becomes a responsibility of the application either to ensure single-threaded handling for a particular
-file path, or rely on some external locking mechanism of its own. Failure to do so will result in
-unexpected behavior.
-
-### Multithread Support
-
-Rename and Delete blob operations on directories with large number of files and sub directories currently is very slow as these operations are done one blob at a time serially. These files and sub folders can be deleted or renamed parallel. Following configurations can be used to enable threads to do parallel processing
-
-To enable 10 threads for Delete operation. Set configuration value to 0 or 1 to disable threads. The default behavior is threads disabled.
-
-```xml
-
- fs.azure.delete.threads
- 10
-
-```
-
-To enable 20 threads for Rename operation. Set configuration value to 0 or 1 to disable threads. The default behavior is threads disabled.
-
-```xml
-
- fs.azure.rename.threads
- 20
-
-```
-
-### WASB Secure mode and configuration
-
-WASB can operate in secure mode where the Storage access keys required to communicate with Azure storage does not have to
-be in the same address space as the process using WASB. In this mode all interactions with Azure storage is performed using
-SAS uris. There are two sub modes within the Secure mode, one is remote SAS key mode where the SAS keys are generated from
-a remote process and local mode where SAS keys are generated within WASB. By default the SAS Key mode is expected to run in
-Romote mode, however for testing purposes the local mode can be enabled to generate SAS keys in the same process as WASB.
-
-To enable Secure mode following property needs to be set to true.
-
-```xml
-
- fs.azure.secure.mode
- true
-
-```
-
-To enable SAS key generation locally following property needs to be set to true.
-
-```xml
-
- fs.azure.local.sas.key.mode
- true
-
-```
-
-To use the remote SAS key generation mode, comma separated external REST services are expected to provided required SAS keys.
-Following property can used to provide the end point to use for remote SAS Key generation:
-
-```xml
-
- fs.azure.cred.service.urls
- {URL}
-
-```
-
-The remote service is expected to provide support for two REST calls ```{URL}/GET_CONTAINER_SAS``` and ```{URL}/GET_RELATIVE_BLOB_SAS```, for generating
-container and relative blob sas keys. An example requests
-
-```{URL}/GET_CONTAINER_SAS?storage_account=&container=&sas_expiry=&delegation_token=```
-```{URL}/GET_CONTAINER_SAS?storage_account=&container=&relative_path=&sas_expiry=&delegation_token=```
-
-The service is expected to return a response in JSON format:
-
-```json
-{
- "responseCode" : 0 or non-zero ,
- "responseMessage" : relavant message on failure ,
- "sasKey" : Requested SAS Key
-}
-```
-
-### Authorization Support in WASB
-
-Authorization support can be enabled in WASB using the following configuration:
-
-```xml
-
- fs.azure.authorization
- true
-
-```
-
-The current implementation of authorization relies on the presence of an external service that can enforce
-the authorization. The service is expected to be running on comma separated URLs provided by the following config.
-
-```xml
-
- fs.azure.authorization.remote.service.urls
- {URL}
-
-```
-
-The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION```
-An example request:
- ```{URL}/CHECK_AUTHORIZATION?wasb_absolute_path=&operation_type=&delegation_token=```
-
-The service is expected to return a response in JSON format:
-
-```json
-{
- "responseCode" : 0 or non-zero ,
- "responseMessage" : relevant message on failure ,
- "authorizationResult" : true/false
-}
-```
-
-### Delegation token support in WASB
-
-Delegation token support can be enabled in WASB using the following configuration:
-
-```xml
-
- fs.azure.enable.kerberos.support
- true
-
-```
-
-The current implementation of delegation token implementation relies on the presence of an external service instances that can generate and manage delegation tokens. The service is expected to be running on comma separated URLs provided by the following config.
-
-```xml
-
- fs.azure.delegation.token.service.urls
- {URL}
-
-```
-
-The remote service is expected to provide support for the following REST call: ```{URL}?op=GETDELEGATIONTOKEN```, ```{URL}?op=RENEWDELEGATIONTOKEN``` and ```{URL}?op=CANCELDELEGATIONTOKEN```
-An example request:
- ```{URL}?op=GETDELEGATIONTOKEN&renewer=```
- ```{URL}?op=RENEWDELEGATIONTOKEN&token=```
- ```{URL}?op=CANCELDELEGATIONTOKEN&token=```
-
-The service is expected to return a response in JSON format for GETDELEGATIONTOKEN request:
-
-```json
-{
- "Token" : {
- "urlString": URL string of delegation token.
- }
-}
-```
-### chown behaviour when authorization is enabled in WASB
-
-When authorization is enabled, only the users listed in the following configuration
-are allowed to change the owning user of files/folders in WASB. The configuration
-value takes a comma separated list of user names who are allowed to perform chown.
-
-```xml
-
- fs.azure.chown.allowed.userlist
- user1,user2
-
-```
-### chmod behaviour when authorization is enabled in WASB
-
-When authorization is enabled, only the owner and the users listed in the
-following configurations are allowed to change the permissions of files/folders in WASB.
-The configuration value takes a comma separated list of user names who are allowed to perform chmod.
-
-```xml
-
- fs.azure.daemon.userlist
- user1,user2
-
-
- fs.azure.chmod.allowed.userlist
- userA,userB
-
-```
-
-Caching of both SAS keys and Authorization responses can be enabled using the following setting:
-The cache settings are applicable only when fs.azure.authorization is enabled.
-The cache is maintained at a filesystem object level.
-```
-
- fs.azure.authorization.caching.enable
- true
-
-```
-
-The maximum number of entries that the cache can hold can be customized using the following setting:
-```
-
- fs.azure.authorization.caching.maxentries
- 512
-
-```
-
- The validity of an authorization cache-entry can be controlled using the following setting:
- Setting the value to zero disables authorization-caching.
- If the key is not specified, a default expiry duration of 5m takes effect.
- ```
-
- fs.azure.authorization.cacheentry.expiry.period
- 5m
-
-```
-
- The validity of a SASKey cache-entry can be controlled using the following setting.
- Setting the value to zero disables SASKey-caching.
- If the key is not specified, the default expiry duration specified in the sas-key request takes effect.
- ```
-
- fs.azure.saskey.cacheentry.expiry.period
- 90d
-
-```
-
- Use container saskey for access to all blobs within the container.
- Blob-specific saskeys are not used when this setting is enabled.
- This setting provides better performance compared to blob-specific saskeys.
- ```
-
- fs.azure.saskey.usecontainersaskeyforallaccess
- true
-
-```
-
-### Performance optimization configurations
-
-`fs.azure.block.blob.buffered.pread.disable`: By default the positional read API will do a
-seek and read on input stream. This read will fill the buffer cache in
-BlockBlobInputStream. If this configuration is true it will skip usage of buffer and do a
-lock free call for reading from blob. This optimization is very much helpful for HBase kind
-of short random read over a shared InputStream instance.
-Note: This is not a config which can be set at cluster level. It can be used as
-an option on FutureDataInputStreamBuilder.
-See FileSystem#openFile(Path path)
-
-## Further Reading
-
-* [Testing the Azure WASB client](testing_azure.html).
-* MSDN article, [Understanding Block Blobs, Append Blobs, and Page Blobs](https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs)
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/fns_blob.md b/hadoop-tools/hadoop-azure/src/site/markdown/fns_blob.md
index 9850bc34170df..44fed0d33f41a 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/fns_blob.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/fns_blob.md
@@ -25,15 +25,15 @@ Refer to [WASB Deprecation](./wasb.html) for more details.
## Azure Service Endpoints Used by ABFS Driver
Azure Services offers two set of endpoints for interacting with storage accounts:
-1. [Azure Blob Storage](./blobEndpoint.md) referred as Blob Endpoint
+1. [Azure Blob Storage](./blobEndpoint.html) referred as Blob Endpoint
2. [Azure Data Lake Storage](https://learn.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/operation-groups) referred as DFS Endpoint
The ABFS Driver by default is designed to work with DFS Endpoint only which primarily
supports HNS Enabled Accounts only.
-To enable ABFS Driver to work with FNS Accounts, support for Blob Endpoint is being added.
+To enable ABFS Driver to work with FNS Accounts, support for Blob Endpoint is added.
This is because Azure services do not recommend using DFS Endpoint for FNS Accounts.
-FNS over DFS endpoint is **REMOVED**. All requests will be switched to Blob endpoint internally if
+FNS over DFS endpoint is therefore **REMOVED**. All requests will be switched to Blob endpoint internally if
account is detected as FNS.
ABFS Driver will only allow FNS Accounts to be accessed using Blob Endpoint.
@@ -90,11 +90,11 @@ configured service type. Choosing a separate ingress service is **only supported
```
- How to configure Shared Key
- auth: [Shared Key](./index.md#a-nameshared-key-autha-default-shared-key)
+ auth: [Shared Key](./index.html#a-nameshared-key-autha-default-shared-key)
- How to configure
- OAuth: [OAuth](./index.md#a-nameoauth-client-credentialsa-oauth-20-client-credentials)
+ OAuth: [OAuth](./index.html#a-nameoauth-client-credentialsa-oauth-20-client-credentials)
- How to configure fixed
- SAS: [Fixed SAS](./index.md#using-accountservice-sas-with-abfs)
+ SAS: [Fixed SAS](./index.html#using-accountservice-sas-with-abfs)
OAuth is recommended auth type as it is more secure and flexible.
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 98ee99256b39f..5a90265c81220 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -33,9 +33,9 @@ You can set this locally in your `.profile`/`.bashrc`, but note it won't
propagate to jobs running in-cluster.
See also:
-* [FNS (non-HNS)](./fns_blob.html)
-* [Legacy-Deprecated-WASB](./wasb.html)
-* [Testing](./testing_azure.html)
+* [ABFS Driver for FNS (non-HNS) Accounts](./fns_blob.html)
+* [Deprecated WASB Driver for FNS (non-HNS) Accounts](./wasb.html)
+* [Testing of ABFS Driver](./testing_azure.html)
* [WASB Migration Config Support](./wasbToAbfsMigration.html)
## Features of the ABFS connector.
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
index f8e4dde3e86e8..fd2445ab550e6 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
@@ -12,7 +12,15 @@
limitations under the License. See accompanying LICENSE file.
-->
-# Testing the Azure WASB client
+# Testing the Azure ABFS client
+
+Azure Data Lake Storage Gen 2 (ADLS Gen 2) is a set of capabilities dedicated to
+big data analytics, built on top of Azure Blob Storage. The ABFS and ABFSS
+schemes target the ADLS Gen 2 REST API now having support for both HNS and FNS Accounts.
+ADLS Gen 2 with HNS Enabled using DFS Endpoint offers better performance and
+scalability. ADLS Gen 2 also offers authentication and authorization compatible
+with the Hadoop Distributed File System permissions model when hierarchical
+namespace is enabled for the storage account.
@@ -24,7 +32,7 @@ convention `Test*.java`. Integration tests follow the naming convention
## Policy for submitting patches which affect the `hadoop-azure` module.
-The Apache Jenkins infrastucture does not run any cloud integration tests,
+The Apache Jenkins infrastructure does not run any cloud integration tests,
due to the need to keep credentials secure.
### The submitter of any patch is required to run all the integration tests and declare which Azure region they used.
@@ -90,17 +98,17 @@ For example:
- fs.azure.wasb.account.name
- {ACCOUNTNAME}.blob.core.windows.net
+ fs.azure.abfs.account.name
+ {ACCOUNTNAME}.dfs.core.windows.net
- fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net
+ fs.azure.account.key.{ACCOUNTNAME}.dfs.core.windows.net
{ACCOUNT ACCESS KEY}
```
-To run contract tests, set the WASB file system URI in `src/test/resources/azure-auth-keys.xml`
+To run contract tests, set the ABFS file system URI in `src/test/resources/azure-auth-keys.xml`
and the account access key. For example:
```xml
@@ -108,12 +116,12 @@ and the account access key. For example:
- fs.contract.test.fs.wasb
- wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net
+ fs.contract.test.fs.abfs
+ wasb://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net
The name of the azure file system for testing.
- fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net
+ fs.azure.account.key.{ACCOUNTNAME}.dfs.core.windows.net
{ACCOUNT ACCESS KEY}
@@ -126,21 +134,21 @@ Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml`
- fs.azure.wasb.account.name
- {ACCOUNTNAME}.blob.core.windows.net
+ fs.azure.abfs.account.name
+ {ACCOUNTNAME}.dfs.core.windows.net
- fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net
+ fs.azure.account.key.{ACCOUNTNAME}.dfs.core.windows.net
{ACCOUNT ACCESS KEY}
- fs.contract.test.fs.wasb
- wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net
+ fs.contract.test.fs.abfs
+ wasb://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net
```
-DO NOT ADD `azure-auth-keys.xml` TO REVISION CONTROL. The keys to your Azure
+DO NOT ADD `azure-auth-keys.xml` TO VERSION CONTROL. The keys to your Azure
Storage account are a secret and must not be shared.
@@ -153,35 +161,29 @@ mvn -T 1C clean verify
```
It's also possible to execute multiple test suites in parallel by passing the
-`parallel-tests=wasb|abfs|both` property on the command line. The tests spend most of their
+`parallel-tests=abfs` property on the command line. The tests spend most of their
time blocked on network I/O, so running in parallel tends to
complete full test runs faster.
```bash
-mvn -T 1C -Dparallel-tests=both clean verify
-mvn -T 1C -Dparallel-tests=wasb clean verify
mvn -T 1C -Dparallel-tests=abfs clean verify
```
-`-Dparallel-tests=wasb` runs the WASB related integration tests from azure directory
`-Dparallel-tests=abfs` runs the ABFS related integration tests from azurebfs directory
-`-Dparallel-tests=both` runs all the integration tests from both azure and azurebfs directory
Some tests must run with exclusive access to the storage container, so even with the
`parallel-tests` property, several test suites will run in serial in a separate
Maven execution step after the parallel tests.
-By default, `parallel-tests` runs 4 test suites concurrently. This can be tuned
+By default, `parallel-tests` runs 8 test suites concurrently. This can be tuned
by passing the `testsThreadCount` property.
```bash
mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean verify
```
-
```bash
mvn -T 1C clean test
@@ -266,37 +268,6 @@ The most bandwidth intensive tests (those which upload data) always run
sequentially; those which are slow due to HTTPS setup costs or server-side
actions are included in the set of parallelized tests.
-
-### Scale test tuning options
-
-
-Some of the tests can be tuned from the maven build or from the
-configuration file used to run the tests.
-
-```bash
-mvn -T 1C verify -Dparallel-tests -Dscale -DtestsThreadCount=8 -Dfs.azure.scale.test.huge.filesize=128M
-```
-
-The algorithm is
-
-1. The value is queried from the configuration file, using a default value if
-it is not set.
-1. The value is queried from the JVM System Properties, where it is passed
-down by maven.
-1. If the system property is null, an empty string, or it has the value `unset`,
-then the configuration value is used. The `unset` option is used to
-[work round a quirk in maven property propagation](http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven).
-
-Only a few properties can be set this way; more will be added.
-
-| Property | Meaninging |
-|-----------|-------------|
-| `fs.azure.scale.test.huge.filesize`| Size for huge file uploads |
-| `fs.azure.scale.test.huge.huge.partitionsize`| Size for partitions in huge file uploads |
-
-The file and partition sizes are numeric values with a k/m/g/t/p suffix depending
-on the desired size. For example: 128M, 128m, 2G, 2G, 4T or even 1P.
-
#### Scale test configuration options
Some scale tests perform multiple operations (such as creating many directories).
@@ -340,23 +311,6 @@ smaller to achieve faster test runs.
```
-Azure-specific scale test properties are
-
-##### `fs.azure.scale.test.huge.filesize`: size in MB for "Huge file tests".
-
-The Huge File tests validate Azure storages's ability to handle large files —the property
-`fs.azure.scale.test.huge.filesize` declares the file size to use.
-
-```xml
-
- fs.azure.scale.test.huge.filesize
- 200M
-
-```
-
-Tests at this scale are slow: they are best executed from hosts running in
-the cloud infrastructure where the storage endpoint is based.
-
## Using the emulator
A selection of tests can run against the
@@ -389,7 +343,7 @@ Logging at debug level is the standard way to provide more diagnostics output;
after setting this rerun the tests
```properties
-log4j.logger.org.apache.hadoop.fs.azure=DEBUG
+log4j.logger.org.apache.hadoop.fs.azurebfs=DEBUG
```
## Adding new tests
@@ -415,18 +369,6 @@ call to `exists()`, `isFile()`, etc.
on a failure. Using `org.apache.hadoop.fs.contract.ContractTestUtils` to make
assertions about the state of a filesystem helps here.
-*Isolating Scale tests*. Any test doing large amounts of IO MUST extend the
-class `AbstractAzureScaleTest`, so only running if `scale` is defined on a build,
-supporting test timeouts configurable by the user. Scale tests should also
-support configurability as to the actual size of objects/number of operations,
-so that behavior at different scale can be verified.
-
-*Designed for parallel execution*. A key need here is for each test suite to work
-on isolated parts of the filesystem. Subclasses of `AbstractWasbTestBase`
-SHOULD use the `path()`, `methodpath()` and `blobpath()` methods,
-to build isolated paths. Tests MUST NOT assume that they have exclusive access
-to a bucket.
-
*Extending existing tests where appropriate*. This recommendation goes
against normal testing best practise of "test one thing per method".
Because it is so slow to create directory trees or upload large files, we do
@@ -453,31 +395,8 @@ is critical.
There are a set of base classes which should be extended for Azure tests and
integration tests.
-##### `org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout`
-
-This extends the junit `Assert` class with thread names and timeouts,
-the default timeout being set in `AzureTestConstants.AZURE_TEST_TIMEOUT` to
-ten minutes. The thread names are set to aid analyzing the stack trace of
-a test: a `jstack` call can be used to
-
-##### `org.apache.hadoop.fs.azure.AbstractWasbTestBase`
-
-The base class for tests which use `AzureBlobStorageTestAccount` to create
-mock or live Azure clients; in test teardown it tries to clean up store state.
-
-1. This class requires subclasses to implement `createTestAccount()` to create
-a mock or real test account.
-
-1. The configuration used to create a test account *should* be that from
-`createConfiguration()`; this can be extended in subclasses to tune the settings.
-
-
-##### `org.apache.hadoop.fs.azure.integration.AbstractAzureScaleTest`
-
-This extends `AbstractWasbTestBase` for scale tests; those test which
-only run when `-Dscale` is used to select the "scale" profile.
-These tests have a timeout of 30 minutes, so as to support slow test runs.
-
+##### `org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest`
+This is the base class for all ABFS integration tests.
Having shared base classes help reduces future maintenance. Please
use them.
@@ -491,31 +410,17 @@ not provide meaningful logs or assertion messages precisely to avoid this.
This means efficient in test setup/teardown, and, ideally, making use of
existing public datasets to save setup time and tester cost.
-
-The reference example is `ITestAzureHugeFiles`:. This marks the test suite as
-`@FixMethodOrder(MethodSorters.NAME_ASCENDING)` then orders the test cases such
-that each test case expects the previous test to have completed (here: uploaded a file,
-renamed a file, ...). This provides for independent tests in the reports, yet still
-permits an ordered sequence of operations. Do note the use of `Assume.assume()`
-to detect when the preconditions for a single test case are not met, hence,
-the tests become skipped, rather than fail with a trace which is really a false alarm.
-
-
### Works Over Long-haul Links
-As well as making file size and operation counts scaleable, this includes
-making test timeouts adequate. The Scale tests make this configurable; it's
-hard coded to ten minutes in `AbstractAzureIntegrationTest()`; subclasses can
-change this by overriding `getTestTimeoutMillis()`.
+As well as making file size and operation counts scalable, this includes
+making test timeouts adequate.
-Equally importantly: support proxies, as some testers need them.
+Equally, importantly: support proxies, as some testers need them.
### Provides Diagnostics and timing information
1. Create logs, log things.
-1. you can use `AbstractWasbTestBase.describe(format-string, args)` here; it
-adds some newlines so as to be easier to spot.
1. Use `ContractTestUtils.NanoTimer` to measure the duration of operations,
and log the output.
@@ -535,7 +440,7 @@ including error messages*.
Keeps costs down.
-1. Do not only cleanup if a test case completes successfully; test suite
+1. Do not only clean up if a test case completes successfully; test suite
teardown must do it.
1. That teardown code must check for the filesystem and other fields being
null before the cleanup. Why? If test setup fails, the teardown methods still
@@ -550,7 +455,7 @@ We really appreciate this — you will too.
### How to keep your credentials really safe
-Although the `auth-keys.xml` file is marged as ignored in git and subversion,
+Although the `auth-keys.xml` file is marked as ignored in git and subversion,
it is still in your source tree, and there's always that risk that it may
creep out.
@@ -568,7 +473,7 @@ using an absolute XInclude reference to it.
### Cleaning up Containers
-The Azure tests create containers with the prefix `"wasbtests-"` and delete
+The Azure tests create containers with the prefix `"abfs-testcontainer-"` and delete
them after the test runs. If a test run is interrupted, these containers
may not get deleted. There is a special test case which can be manually invoked
to list and delete these, `CleanupTestContainers`
@@ -581,17 +486,6 @@ This will delete the containers; the output log of the test run will
provide the details and summary of the operation.
-# Testing the Azure ABFS Client
-
-Azure Data Lake Storage Gen 2 (ADLS Gen 2) is a set of capabilities dedicated to
-big data analytics, built on top of Azure Blob Storage. The ABFS and ABFSS
-schemes target the ADLS Gen 2 REST API, and the WASB and WASBS schemes target
-the Azure Blob Storage REST API. ADLS Gen 2 offers better performance and
-scalability. ADLS Gen 2 also offers authentication and authorization compatible
-with the Hadoop Distributed File System permissions model when hierarchical
-namespace is enabled for the storage account. Furthermore, the metadata and data
-produced by ADLS Gen 2 REST API can be consumed by Blob REST API, and vice versa.
-
## Generating test run configurations and test triggers over various config combinations
To simplify the testing across various authentication and features combinations
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/wasb.md b/hadoop-tools/hadoop-azure/src/site/markdown/wasb.md
index 270fd14da4c44..571d3e8cdce00 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/wasb.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/wasb.md
@@ -12,17 +12,19 @@
limitations under the License. See accompanying LICENSE file.
-->
-# Hadoop Azure Support: WASB Driver
+# Deprecated WASB Driver
+
+### Note: WASB Driver is removed and won't be a part of official hadoop releases starting from hadoop-3.5.0
## Introduction
-WASB Driver is a legacy Hadoop File System driver that was developed to support
+WASB Driver was a legacy Hadoop File System driver that was developed to support
[FNS(FlatNameSpace) Azure Storage accounts](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction)
that do not honor File-Folder syntax.
-HDFS Folder operations hence are mimicked at client side by WASB driver and
-certain folder operations like Rename and Delete can lead to a lot of IOPs with
+HDFS Folder operations hence were mimicked at client side by WASB driver and
+certain folder operations like Rename and Delete could lead to a lot of IOPs with
client-side enumeration and orchestration of rename/delete operation blob by blob.
It was not ideal for other APIs too as initial checks for path is a file or folder
-needs to be done over multiple metadata calls. These led to a degraded performance.
+needed to be done over multiple metadata calls. These led to a degraded performance.
To provide better service to Analytics users, Microsoft released [ADLS Gen2](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction)
which are HNS (Hierarchical Namespace) enabled, i.e. File-Folder aware storage accounts.
@@ -40,19 +42,19 @@ transition situation.
workloads to migrate to the ABFS driver, which is available only on HNS enabled
accounts in a fully tested and supported scenario.
-## Deprecation plans for WASB Driver
-We are introducing a new feature that will enable the ABFS driver to support
+## Deprecation of WASB Driver
+We have introduced a new feature that will enable the [ABFS](./fns_blob.html) driver to support
FNS accounts (over BlobEndpoint that WASB Driver uses) using the ABFS scheme.
This feature will enable us to use the ABFS driver to interact with data stored in GPv2
(General Purpose v2) storage accounts.
-With this feature, the users who still use the legacy WASB driver will be able
-to migrate to the ABFS driver without much re-work on their workloads. They will
+With this feature, the WASB users are now required to migrate to the ABFS driver
+without much re-work on their workloads. They will
however need to change the URIs from the WASB scheme to the ABFS scheme.
+Refer to [Wasb To Abfs Migration Guide](./wasbToAbfsMigration.html) for more details.
-Once ABFS driver has built FNS support capability to migrate WASB users, WASB
-driver will be marked for removal in next major release. This will remove any ambiguity
-for new users onboards as there will be only one Microsoft driver for Azure Storage
+With removal of WASB Driver our aim is to remove any ambiguity
+for new user onboards as there will be only one Microsoft driver for Azure Storage
and migrating users will get SLA bound support for driver and service,
which was not guaranteed over WASB.
@@ -61,7 +63,7 @@ move to HNS enabled accounts with the ABFS driver, which is our recommended stac
for big data analytics on ADLS Gen2.
### Impact for existing ABFS users using ADLS Gen2 (HNS enabled account)
-This feature does not impact the existing users who are using ADLS Gen2 Accounts
+Removal of WASB Driver does not impact the existing users who are using ADLS Gen2 Accounts
(HNS enabled account) with ABFS driver.
They do not need to make any changes to their workloads or configurations. They
@@ -76,7 +78,7 @@ users to transition to a supported scenario immediately, while they plan to
ultimately move to ADLS Gen2 (HNS enabled account).
### New Authentication Options for a migrating user
-Below auth types that WASB provides will continue to work on the new FNS over
+Below auth types that WASB provided will continue to work on the new FNS over
ABFS Driver over configuration that accepts these SAS types (similar to WASB):
1. SharedKey
2. Account SAS
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/wasbToAbfsMigration.md b/hadoop-tools/hadoop-azure/src/site/markdown/wasbToAbfsMigration.md
index 87bb56083a6fd..9613696ce2cf0 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/wasbToAbfsMigration.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/wasbToAbfsMigration.md
@@ -21,13 +21,13 @@ for the same.
## Introduction
ABFS driver has now built support for
-FNS accounts (over BlobEndpoint that WASB Driver uses) using the ABFS scheme.
+FNS accounts (over BlobEndpoint that WASB Driver used to have) using the ABFS scheme.
Refer to: [ABFS Driver for Namespace Disabled Accounts](./fns_blob.html) for more details.
-The legacy WASB driver has been **deprecated** and is no longer recommended for
-use. Refer to: [WASB Deprecation](./wasb.html) for more details.
+The legacy WASB driver has been **removed** and is no longer part of official hadoop releases.
+Refer to: [WASB Deprecation](./wasb.html) for more details.
It's highly recommended for current WASB Driver users to migrate to ABFS driver,
-the only Microsoft driver for Azure Storage.
+the only Hadoop driver for Azure Storage from Microsoft.
Microsoft recommends all Big Data and Analytics users to use
Azure Data Lake Gen2 (ADLS Gen2) using the ABFS driver. It is thus preferred to
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
deleted file mode 100644
index d963b14d5a01b..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
-import org.apache.hadoop.io.IOUtils;
-
-import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.*;
-
-/**
- * Abstract test class that provides basic setup and teardown of testing Azure
- * Storage account. Each subclass defines a different set of test cases to run
- * and overrides {@link #createTestAccount()} to set up the testing account used
- * to run those tests. The returned account might integrate with Azure Storage
- * directly or it might be a mock implementation.
- */
-public abstract class AbstractWasbTestBase extends AbstractWasbTestWithTimeout
- implements AzureTestConstants {
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(AbstractWasbTestBase.class);
-
- protected NativeAzureFileSystem fs;
- protected AzureBlobStorageTestAccount testAccount;
-
- @BeforeEach
- public void setUp() throws Exception {
- AzureBlobStorageTestAccount account = createTestAccount();
- assumeNotNull(account, "test account");
- bindToTestAccount(account);
- }
-
- @AfterEach
- public void tearDown() throws Exception {
- describe("closing test account and filesystem");
- testAccount = cleanupTestAccount(testAccount);
- IOUtils.closeStream(fs);
- fs = null;
- }
-
- /**
- * Create the configuration to use when creating a test account.
- * Subclasses can override this to tune the test account configuration.
- * @return a configuration.
- */
- public Configuration createConfiguration() {
- return AzureBlobStorageTestAccount.createTestConfiguration();
- }
-
- /**
- * Create the test account.
- * Subclasses must implement this.
- * @return the test account.
- * @throws Exception
- */
- protected abstract AzureBlobStorageTestAccount createTestAccount()
- throws Exception;
-
- /**
- * Get the test account.
- * @return the current test account.
- */
- protected AzureBlobStorageTestAccount getTestAccount() {
- return testAccount;
- }
-
- /**
- * Get the filesystem
- * @return the current filesystem.
- */
- protected NativeAzureFileSystem getFileSystem() {
- return fs;
- }
-
- /**
- * Get the configuration used to create the filesystem
- * @return the configuration of the test FS
- */
- protected Configuration getConfiguration() {
- return getFileSystem().getConf();
- }
-
- /**
- * Bind to a new test account; closing any existing one.
- * This updates the test account returned in {@link #getTestAccount()}
- * and the filesystem in {@link #getFileSystem()}.
- * @param account new test account
- */
- protected void bindToTestAccount(AzureBlobStorageTestAccount account) {
- // clean any existing test account
- cleanupTestAccount(testAccount);
- IOUtils.closeStream(fs);
- testAccount = account;
- if (testAccount != null) {
- fs = testAccount.getFileSystem();
- }
- }
-
- /**
- * Return a path to a blob which will be unique for this fork.
- * @param filepath filepath
- * @return a path under the default blob directory
- * @throws IOException
- */
- protected Path blobPath(String filepath) throws IOException {
- return blobPathForTests(getFileSystem(), filepath);
- }
-
- /**
- * Create a path under the test path provided by
- * the FS contract.
- * @param filepath path string in
- * @return a path qualified by the test filesystem
- * @throws IOException IO problems
- */
- protected Path path(String filepath) throws IOException {
- return pathForTests(getFileSystem(), filepath);
- }
-
- /**
- * Return a path bonded to this method name, unique to this fork during
- * parallel execution.
- * @return a method name unique to (fork, method).
- * @throws IOException IO problems
- */
- protected Path methodPath() throws IOException {
- return path(methodName.getMethodName());
- }
-
- /**
- * Return a blob path bonded to this method name, unique to this fork during
- * parallel execution.
- * @return a method name unique to (fork, method).
- * @throws IOException IO problems
- */
- protected Path methodBlobPath() throws IOException {
- return blobPath(methodName.getMethodName());
- }
-
- /**
- * Describe a test in the logs.
- * @param text text to print
- * @param args arguments to format in the printing
- */
- protected void describe(String text, Object... args) {
- LOG.info("\n\n{}: {}\n",
- methodName.getMethodName(),
- String.format(text, args));
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
deleted file mode 100644
index d6624e8c0c178..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Timeout;
-import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
-import org.junit.jupiter.api.extension.RegisterExtension;
-import org.apache.hadoop.test.TestName;
-
-import java.util.concurrent.TimeUnit;
-
-import static org.assertj.core.api.Assumptions.assumeThat;
-
-/**
- * Base class for any Wasb test with timeouts & named threads.
- * This class does not attempt to bind to Azure.
- */
-@Timeout(value = AzureTestConstants.AZURE_TEST_TIMEOUT, unit = TimeUnit.MILLISECONDS)
-public class AbstractWasbTestWithTimeout extends Assertions {
-
- /**
- * The name of the current method.
- */
- @RegisterExtension
- public TestName methodName = new TestName();
-
- /**
- * Name the junit thread for the class. This will overridden
- * before the individual test methods are run.
- */
- @BeforeAll
- public static void nameTestThread() {
- Thread.currentThread().setName("JUnit");
- }
-
- /**
- * Name the thread to the current test method.
- */
- @BeforeEach
- public void nameThread() {
- Thread.currentThread().setName("JUnit-" + methodName.getMethodName());
- }
-
- /**
- * Override point: the test timeout in milliseconds.
- * @return a timeout in milliseconds
- */
- protected int getTestTimeoutMillis() {
- return AzureTestConstants.AZURE_TEST_TIMEOUT;
- }
-
- public static void assumeNotNull(Object objects) {
- assumeThat(objects).isNotNull();
- }
-
- public static void assumeNotNull(Object objects, String message) {
- assumeThat(objects).as(message).isNotNull();
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
deleted file mode 100644
index c5f6cb762a337..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ /dev/null
@@ -1,947 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-package org.apache.hadoop.fs.azure;
-
-import com.microsoft.azure.storage.*;
-import com.microsoft.azure.storage.blob.*;
-import com.microsoft.azure.storage.core.Base64;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.configuration2.SubsetConfiguration;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.metrics2.AbstractMetric;
-import org.apache.hadoop.metrics2.MetricsRecord;
-import org.apache.hadoop.metrics2.MetricsSink;
-import org.apache.hadoop.metrics2.MetricsTag;
-import org.apache.hadoop.metrics2.impl.TestMetricsConfig;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.*;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME;
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCAL_SAS_KEY_MODE;
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
-import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-/**
- * Helper class to create WASB file systems backed by either a mock in-memory
- * implementation or a real Azure Storage account.
- */
-public final class AzureBlobStorageTestAccount implements AutoCloseable,
- AzureTestConstants {
- private static final Logger LOG = LoggerFactory.getLogger(
- AzureBlobStorageTestAccount.class);
-
- private static final String SAS_PROPERTY_NAME = "fs.azure.sas.";
- private static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
- public static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
- public static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name";
- public static final String WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN = "fs.azure.wasb.account.name";
- public static final String MOCK_ACCOUNT_NAME = "mockAccount-c01112a3-2a23-433e-af2a-e808ea385136.blob.core.windows.net";
- public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX = ".blob.core.windows.net";
- public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX = "\\.blob(\\.preprod)?\\.core\\.windows\\.net";
- public static final String MOCK_CONTAINER_NAME = "mockContainer";
- public static final String WASB_AUTHORITY_DELIMITER = "@";
- public static final String WASB_SCHEME = "wasb";
- public static final String PATH_DELIMITER = "/";
- public static final String AZURE_ROOT_CONTAINER = "$root";
- public static final String MOCK_WASB_URI = "wasb://" + MOCK_CONTAINER_NAME
- + WASB_AUTHORITY_DELIMITER + MOCK_ACCOUNT_NAME + "/";
- private static final String USE_EMULATOR_PROPERTY_NAME = "fs.azure.test.emulator";
-
- private static final String KEY_DISABLE_THROTTLING = "fs.azure.disable.bandwidth.throttling";
- private static final String KEY_READ_TOLERATE_CONCURRENT_APPEND = "fs.azure.io.read.tolerate.concurrent.append";
- public static final String DEFAULT_PAGE_BLOB_DIRECTORY = "pageBlobs";
- public static final String DEFAULT_ATOMIC_RENAME_DIRECTORIES = "/atomicRenameDir1,/atomicRenameDir2";
-
- private CloudStorageAccount account;
- private CloudBlobContainer container;
- private CloudBlockBlob blob;
- private NativeAzureFileSystem fs;
- private AzureNativeFileSystemStore storage;
- private MockStorageInterface mockStorage;
- private String pageBlobDirectory;
- private static final ConcurrentLinkedQueue allMetrics =
- new ConcurrentLinkedQueue();
- private static boolean metricsConfigSaved = false;
- private boolean skipContainerDelete = false;
-
- private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
- CloudStorageAccount account,
- CloudBlobContainer container) {
- this(fs, account, container, false);
- }
-
- private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
- CloudStorageAccount account,
- CloudBlobContainer container,
- boolean skipContainerDelete) {
- this.account = account;
- this.container = container;
- this.fs = fs;
- this.skipContainerDelete = skipContainerDelete;
- }
-
- /**
- * Create a test account with an initialized storage reference.
- *
- * @param storage
- * -- store to be accessed by the account
- * @param account
- * -- Windows Azure account object
- * @param container
- * -- Windows Azure container object
- */
- private AzureBlobStorageTestAccount(AzureNativeFileSystemStore storage,
- CloudStorageAccount account, CloudBlobContainer container) {
- this.account = account;
- this.container = container;
- this.storage = storage;
- }
-
- /**
- * Create a test account sessions with the default root container.
- *
- * @param fs
- * - file system, namely WASB file system
- * @param account
- * - Windows Azure account object
- * @param blob
- * - block blob reference
- */
- private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
- CloudStorageAccount account, CloudBlockBlob blob) {
-
- this.account = account;
- this.blob = blob;
- this.fs = fs;
- }
-
- private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
- MockStorageInterface mockStorage) {
- this.fs = fs;
- this.mockStorage = mockStorage;
- }
-
- private static void addRecord(MetricsRecord record) {
- allMetrics.add(record);
- }
-
- public static String getMockContainerUri() {
- return String.format("http://%s/%s",
- AzureBlobStorageTestAccount.MOCK_ACCOUNT_NAME,
- AzureBlobStorageTestAccount.MOCK_CONTAINER_NAME);
- }
-
- public static String toMockUri(String path) {
- return String.format("http://%s/%s/%s",
- AzureBlobStorageTestAccount.MOCK_ACCOUNT_NAME,
- AzureBlobStorageTestAccount.MOCK_CONTAINER_NAME, path);
- }
-
- public static String toMockUri(Path path) {
- // Remove the first SEPARATOR
- return toMockUri(path.toUri().getRawPath().substring(1));
- }
-
- public static Path pageBlobPath() {
- return new Path("/" + DEFAULT_PAGE_BLOB_DIRECTORY);
- }
-
- @Deprecated
- public static Path pageBlobPath(String fileName) {
- return new Path(pageBlobPath(), fileName);
- }
-
- public Number getLatestMetricValue(String metricName, Number defaultValue)
- throws IndexOutOfBoundsException{
- boolean found = false;
- Number ret = null;
- for (MetricsRecord currentRecord : allMetrics) {
- // First check if this record is coming for my file system.
- if (wasGeneratedByMe(currentRecord)) {
- for (AbstractMetric currentMetric : currentRecord.metrics()) {
- if (currentMetric.name().equalsIgnoreCase(metricName)) {
- found = true;
- ret = currentMetric.value();
- break;
- }
- }
- }
- }
- if (!found) {
- if (defaultValue != null) {
- return defaultValue;
- }
- throw new IndexOutOfBoundsException(metricName);
- }
- return ret;
- }
-
- /**
- * Checks if the given record was generated by my WASB file system instance.
- * @param currentRecord The metrics record to check.
- * @return
- */
- private boolean wasGeneratedByMe(MetricsRecord currentRecord) {
- assertNotNull(fs, "null filesystem");
- assertNotNull(fs.getInstrumentation().getFileSystemInstanceId(),
- "null filesystemn instance ID");
- String myFsId = fs.getInstrumentation().getFileSystemInstanceId().toString();
- for (MetricsTag currentTag : currentRecord.tags()) {
- if (currentTag.name().equalsIgnoreCase("wasbFileSystemId")) {
- return currentTag.value().equals(myFsId);
- }
- }
- return false;
- }
-
-
- /**
- * Gets the blob reference to the given blob key.
- *
- * @param blobKey
- * The blob key (no initial slash).
- * @return The blob reference.
- */
- public CloudBlockBlob getBlobReference(String blobKey)
- throws Exception {
- return container.getBlockBlobReference(
- String.format(blobKey));
- }
-
- /**
- * Acquires a short lease on the given blob in this test account.
- *
- * @param blobKey
- * The key to the blob (no initial slash).
- * @return The lease ID.
- */
- public String acquireShortLease(String blobKey) throws Exception {
- return getBlobReference(blobKey).acquireLease(60, null);
- }
-
- /**
- * Releases the lease on the container.
- *
- * @param leaseID
- * The lease ID.
- */
- public void releaseLease(String leaseID, String blobKey) throws Exception {
- AccessCondition accessCondition = new AccessCondition();
- accessCondition.setLeaseID(leaseID);
- getBlobReference(blobKey).releaseLease(accessCondition);
- }
-
- private static void saveMetricsConfigFile() throws IOException {
- if (!metricsConfigSaved) {
- String testFilename = TestMetricsConfig.getTestFilename(
- "hadoop-metrics2-azure-file-system");
- File dest = new File(testFilename).getCanonicalFile();
- dest.getParentFile().mkdirs();
- new org.apache.hadoop.metrics2.impl.ConfigBuilder()
- .add("azure-file-system.sink.azuretestcollector.class",
- StandardCollector.class.getName())
- .save(testFilename);
- metricsConfigSaved = true;
- }
- }
-
- public static AzureBlobStorageTestAccount createMock() throws Exception {
- return createMock(new Configuration());
- }
-
- public static AzureBlobStorageTestAccount createMock(Configuration conf) throws Exception {
- saveMetricsConfigFile();
- configurePageBlobDir(conf);
- configureAtomicRenameDir(conf);
- AzureNativeFileSystemStore store = new AzureNativeFileSystemStore();
- MockStorageInterface mockStorage = new MockStorageInterface();
- store.setAzureStorageInteractionLayer(mockStorage);
- NativeAzureFileSystem fs = new NativeAzureFileSystem(store);
- setMockAccountKey(conf);
- configureSecureModeTestSettings(conf);
- // register the fs provider.
-
- fs.initialize(new URI(MOCK_WASB_URI), conf);
- AzureBlobStorageTestAccount testAcct =
- new AzureBlobStorageTestAccount(fs, mockStorage);
- return testAcct;
- }
-
- /**
- * Set the page blob directories configuration to the default if it is not
- * already set. Some tests may set it differently (e.g. the page blob
- * tests in TestNativeAzureFSPageBlobLive).
- * @param conf The configuration to conditionally update.
- */
- private static void configurePageBlobDir(Configuration conf) {
- if (conf.get(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES) == null) {
- conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES,
- "/" + DEFAULT_PAGE_BLOB_DIRECTORY);
- }
- }
-
- /** Do the same for the atomic rename directories configuration */
- private static void configureAtomicRenameDir(Configuration conf) {
- if (conf.get(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES) == null) {
- conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES,
- DEFAULT_ATOMIC_RENAME_DIRECTORIES);
- }
- }
-
- /**
- * Creates a test account that goes against the storage emulator.
- *
- * @return The test account, or null if the emulator isn't setup.
- */
- public static AzureBlobStorageTestAccount createForEmulator()
- throws Exception {
- saveMetricsConfigFile();
- NativeAzureFileSystem fs = null;
- CloudBlobContainer container = null;
- Configuration conf = createTestConfiguration();
- if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) {
- // Not configured to test against the storage emulator.
- LOG.warn("Skipping emulator Azure test because configuration "
- + "doesn't indicate that it's running.");
- return null;
- }
- CloudStorageAccount account =
- CloudStorageAccount.getDevelopmentStorageAccount();
- fs = new NativeAzureFileSystem();
- String containerName = String.format("wasbtests-%s-%tQ",
- System.getProperty("user.name"), new Date());
- container = account.createCloudBlobClient().getContainerReference(
- containerName);
- container.create();
-
- // Set account URI and initialize Azure file system.
- URI accountUri = createAccountUri(DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME,
- containerName);
- configureSecureModeTestSettings(conf);
-
- fs.initialize(accountUri, conf);
-
- // Create test account initializing the appropriate member variables.
- //
- AzureBlobStorageTestAccount testAcct =
- new AzureBlobStorageTestAccount(fs, account, container);
-
- return testAcct;
- }
-
- public static AzureBlobStorageTestAccount createOutOfBandStore(
- int uploadBlockSize, int downloadBlockSize) throws Exception {
- return createOutOfBandStore(uploadBlockSize, downloadBlockSize, false);
- }
-
- public static AzureBlobStorageTestAccount createOutOfBandStore(
- int uploadBlockSize, int downloadBlockSize, boolean enableSecureMode) throws Exception {
-
- saveMetricsConfigFile();
-
- CloudBlobContainer container = null;
- Configuration conf = createTestConfiguration();
- CloudStorageAccount account = createTestAccount(conf);
- if (null == account) {
- return null;
- }
-
- String containerName = String.format("wasbtests-%s-%tQ",
- System.getProperty("user.name"), new Date());
-
- // Create the container.
- container = account.createCloudBlobClient().getContainerReference(
- containerName);
- container.create();
-
- String accountName = verifyWasbAccountNameInConfig(conf);
-
- // Ensure that custom throttling is disabled and tolerate concurrent
- // out-of-band appends.
- conf.setBoolean(KEY_DISABLE_THROTTLING, true);
- conf.setBoolean(KEY_READ_TOLERATE_CONCURRENT_APPEND, true);
- conf.setBoolean(KEY_USE_SECURE_MODE, enableSecureMode);
- configureSecureModeTestSettings(conf);
-
- // Set account URI and initialize Azure file system.
- URI accountUri = createAccountUri(accountName, containerName);
-
- // Set up instrumentation.
- //
- AzureFileSystemMetricsSystem.fileSystemStarted();
- String sourceName = NativeAzureFileSystem.newMetricsSourceName();
- String sourceDesc = "Azure Storage Volume File System metrics";
-
- AzureFileSystemInstrumentation instrumentation = new AzureFileSystemInstrumentation(conf);
-
- AzureFileSystemMetricsSystem.registerSource(
- sourceName, sourceDesc, instrumentation);
-
-
- // Create a new AzureNativeFileSystemStore object.
- AzureNativeFileSystemStore testStorage = new AzureNativeFileSystemStore();
-
- // Initialize the store with the throttling feedback interfaces.
- testStorage.initialize(accountUri, conf, instrumentation);
-
- // Create test account initializing the appropriate member variables.
- //
- AzureBlobStorageTestAccount testAcct =
- new AzureBlobStorageTestAccount(testStorage, account, container);
-
- return testAcct;
- }
-
- /**
- * Sets the mock account key in the given configuration.
- *
- * @param conf
- * The configuration.
- */
- public static void setMockAccountKey(Configuration conf) {
- setMockAccountKey(conf, MOCK_ACCOUNT_NAME);
- }
-
- /**
- * Configure default values for Secure Mode testing.
- * These values are relevant only when testing in Secure Mode.
- *
- * @param conf
- * The configuration.
- */
- public static void configureSecureModeTestSettings(Configuration conf) {
- conf.set(KEY_USE_LOCAL_SAS_KEY_MODE, "true"); // always use local sas-key mode for testing
- }
-
- /**
- * Sets the mock account key in the given configuration.
- *
- * @param conf
- * The configuration.
- */
- public static void setMockAccountKey(Configuration conf, String accountName) {
- conf.set(ACCOUNT_KEY_PROPERTY_NAME + accountName,
- Base64.encode(new byte[] { 1, 2, 3 }));
- }
-
- private static URI createAccountUri(String accountName)
- throws URISyntaxException {
- return new URI(WASB_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
- + accountName);
- }
-
- private static URI createAccountUri(String accountName, String containerName)
- throws URISyntaxException {
- return new URI(WASB_SCHEME + ":" + PATH_DELIMITER + PATH_DELIMITER
- + containerName + WASB_AUTHORITY_DELIMITER + accountName);
- }
-
- public static AzureBlobStorageTestAccount create() throws Exception {
- return create("");
- }
-
- public static AzureBlobStorageTestAccount create(String containerNameSuffix)
- throws Exception {
- return create(containerNameSuffix,
- EnumSet.of(CreateOptions.CreateContainer));
- }
-
- // Create a test account which uses throttling.
- public static AzureBlobStorageTestAccount createThrottled() throws Exception {
- return create("",
- EnumSet.of(CreateOptions.useThrottling, CreateOptions.CreateContainer));
- }
-
- public static AzureBlobStorageTestAccount create(Configuration conf)
- throws Exception {
- return create("", EnumSet.of(CreateOptions.CreateContainer), conf);
- }
-
- static CloudStorageAccount createStorageAccount(String accountName,
- Configuration conf, boolean allowAnonymous) throws URISyntaxException,
- KeyProviderException {
- String accountKey = AzureNativeFileSystemStore
- .getAccountKeyFromConfiguration(accountName, conf);
- final StorageCredentials credentials;
- if (accountKey == null) {
- if (allowAnonymous) {
- credentials = StorageCredentialsAnonymous.ANONYMOUS;
- } else {
- LOG.warn("Skipping live Azure test because of missing key for"
- + " account '" + accountName + "'.");
- return null;
- }
- } else {
- credentials = new StorageCredentialsAccountAndKey(
- accountName.split("\\.")[0], accountKey);
- }
-
- return new CloudStorageAccount(credentials);
- }
-
- public static Configuration createTestConfiguration() {
- return createTestConfiguration(null);
- }
-
- private static Configuration createTestConfiguration(Configuration conf) {
- if (conf == null) {
- conf = new Configuration();
- }
-
- conf.addResource(TEST_CONFIGURATION_FILE_NAME);
- return conf;
- }
-
- public static CloudStorageAccount createTestAccount()
- throws URISyntaxException, KeyProviderException
- {
- return createTestAccount(createTestConfiguration());
- }
-
- static CloudStorageAccount createTestAccount(Configuration conf)
- throws URISyntaxException, KeyProviderException {
- AzureTestUtils.assumeNamespaceDisabled(conf);
-
- String testAccountName = verifyWasbAccountNameInConfig(conf);
- if (testAccountName == null) {
- LOG.warn("Skipping live Azure test because of missing test account");
- return null;
- }
- return createStorageAccount(testAccountName, conf, false);
- }
-
- public static enum CreateOptions {
- UseSas, Readonly, CreateContainer, useThrottling
- }
-
- public static AzureBlobStorageTestAccount create(String containerNameSuffix,
- EnumSet createOptions) throws Exception {
- return create(containerNameSuffix, createOptions, null);
- }
-
- public static AzureBlobStorageTestAccount create(
- String containerNameSuffix,
- EnumSet createOptions,
- Configuration initialConfiguration)
- throws Exception {
- return create(containerNameSuffix, createOptions, initialConfiguration, false);
- }
-
- public static AzureBlobStorageTestAccount create(
- String containerNameSuffix,
- EnumSet createOptions,
- Configuration initialConfiguration,
- boolean useContainerSuffixAsContainerName)
- throws Exception {
- saveMetricsConfigFile();
- NativeAzureFileSystem fs = null;
- CloudBlobContainer container = null;
- Configuration conf = createTestConfiguration(initialConfiguration);
- configurePageBlobDir(conf);
- configureAtomicRenameDir(conf);
- CloudStorageAccount account = createTestAccount(conf);
- if (account == null) {
- return null;
- }
- fs = new NativeAzureFileSystem();
- String containerName = useContainerSuffixAsContainerName
- ? containerNameSuffix
- : String.format(
- "wasbtests-%s-%s%s",
- System.getProperty("user.name"),
- UUID.randomUUID().toString(),
- containerNameSuffix);
- container = account.createCloudBlobClient().getContainerReference(
- containerName);
- if (createOptions.contains(CreateOptions.CreateContainer)) {
- container.createIfNotExists();
- }
- String accountName = verifyWasbAccountNameInConfig(conf);
- if (createOptions.contains(CreateOptions.UseSas)) {
- String sas = generateSAS(container,
- createOptions.contains(CreateOptions.Readonly));
- if (!createOptions.contains(CreateOptions.CreateContainer)) {
- // The caller doesn't want the container to be pre-created,
- // so delete it now that we have generated the SAS.
- container.delete();
- }
- // Remove the account key from the configuration to make sure we don't
- // cheat and use that.
- // but only if not in secure mode, which requires that login
- if (!conf.getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, false)) {
- conf.set(ACCOUNT_KEY_PROPERTY_NAME + accountName, "");
- }
- // Set the SAS key.
- conf.set(SAS_PROPERTY_NAME + containerName + "." + accountName, sas);
- }
-
- // Check if throttling is turned on and set throttling parameters
- // appropriately.
- if (createOptions.contains(CreateOptions.useThrottling)) {
- conf.setBoolean(KEY_DISABLE_THROTTLING, false);
- } else {
- conf.setBoolean(KEY_DISABLE_THROTTLING, true);
- }
-
- configureSecureModeTestSettings(conf);
-
- // Set account URI and initialize Azure file system.
- URI accountUri = createAccountUri(accountName, containerName);
- fs.initialize(accountUri, conf);
-
- // Create test account initializing the appropriate member variables.
- //
- AzureBlobStorageTestAccount testAcct =
- new AzureBlobStorageTestAccount(fs, account, container,
- useContainerSuffixAsContainerName);
-
- return testAcct;
- }
-
- private static String generateContainerName() throws Exception {
- String containerName =
- String.format ("wasbtests-%s-%tQ",
- System.getProperty("user.name"),
- new Date());
- return containerName;
- }
-
- private static String generateSAS(CloudBlobContainer container,
- boolean readonly) throws Exception {
-
- // Create a container if it does not exist.
- container.createIfNotExists();
-
- // Create a new shared access policy.
- SharedAccessBlobPolicy sasPolicy = new SharedAccessBlobPolicy();
-
- // Create a UTC Gregorian calendar value.
- GregorianCalendar calendar = new GregorianCalendar(
- TimeZone.getTimeZone("UTC"));
-
- // Specify the current time as the start time for the shared access
- // signature.
- //
- calendar.setTime(new Date());
- sasPolicy.setSharedAccessStartTime(calendar.getTime());
-
- // Use the start time delta one hour as the end time for the shared
- // access signature.
- calendar.add(Calendar.HOUR, 10);
- sasPolicy.setSharedAccessExpiryTime(calendar.getTime());
-
- if (readonly) {
- // Set READ permissions
- sasPolicy.setPermissions(EnumSet.of(
- SharedAccessBlobPermissions.READ,
- SharedAccessBlobPermissions.LIST));
- } else {
- // Set READ and WRITE permissions.
- //
- sasPolicy.setPermissions(EnumSet.of(
- SharedAccessBlobPermissions.READ,
- SharedAccessBlobPermissions.WRITE,
- SharedAccessBlobPermissions.LIST));
- }
-
- // Create the container permissions.
- BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
-
- // Turn public access to the container off.
- containerPermissions.setPublicAccess(BlobContainerPublicAccessType.OFF);
-
- container.uploadPermissions(containerPermissions);
-
- // Create a shared access signature for the container.
- String sas = container.generateSharedAccessSignature(sasPolicy, null);
- // HACK: when the just generated SAS is used straight away, we get an
- // authorization error intermittently. Sleeping for 1.5 seconds fixes that
- // on my box.
- Thread.sleep(1500);
-
- // Return to caller with the shared access signature.
- return sas;
- }
-
- public static void primePublicContainer(CloudBlobClient blobClient,
- String accountName, String containerName, String blobName, int fileSize)
- throws Exception {
-
- // Create a container if it does not exist. The container name
- // must be lower case.
- CloudBlobContainer container = blobClient
- .getContainerReference(containerName);
-
- container.createIfNotExists();
-
- // Create a new shared access policy.
- SharedAccessBlobPolicy sasPolicy = new SharedAccessBlobPolicy();
-
- // Set READ and WRITE permissions.
- //
- sasPolicy.setPermissions(EnumSet.of(
- SharedAccessBlobPermissions.READ,
- SharedAccessBlobPermissions.WRITE,
- SharedAccessBlobPermissions.LIST,
- SharedAccessBlobPermissions.DELETE));
-
- // Create the container permissions.
- BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
-
- // Turn public access to the container off.
- containerPermissions
- .setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
-
- // Set the policy using the values set above.
- containerPermissions.getSharedAccessPolicies().put("testwasbpolicy",
- sasPolicy);
- container.uploadPermissions(containerPermissions);
-
- // Create a blob output stream.
- CloudBlockBlob blob = container.getBlockBlobReference(blobName);
- BlobOutputStream outputStream = blob.openOutputStream();
-
- outputStream.write(new byte[fileSize]);
- outputStream.close();
- }
-
- public static AzureBlobStorageTestAccount createAnonymous(
- final String blobName, final int fileSize) throws Exception {
-
- NativeAzureFileSystem fs = null;
- CloudBlobContainer container = null;
- Configuration conf = createTestConfiguration(), noTestAccountConf = new Configuration();
-
- // Set up a session with the cloud blob client to generate SAS and check the
- // existence of a container and capture the container object.
- CloudStorageAccount account = createTestAccount(conf);
- if (account == null) {
- return null;
- }
- CloudBlobClient blobClient = account.createCloudBlobClient();
-
- // Capture the account URL and the account name.
- String accountName = verifyWasbAccountNameInConfig(conf);
-
- configureSecureModeTestSettings(conf);
-
- // Generate a container name and create a shared access signature string for
- // it.
- //
- String containerName = generateContainerName();
-
- // Set up public container with the specified blob name.
- primePublicContainer(blobClient, accountName, containerName, blobName,
- fileSize);
-
- // Capture the blob container object. It should exist after generating the
- // shared access signature.
- container = blobClient.getContainerReference(containerName);
- if (null == container || !container.exists()) {
- final String errMsg = String
- .format("Container '%s' expected but not found while creating SAS account.");
- throw new Exception(errMsg);
- }
-
- // Set the account URI.
- URI accountUri = createAccountUri(accountName, containerName);
-
- // Initialize the Native Azure file system with anonymous credentials.
- fs = new NativeAzureFileSystem();
- fs.initialize(accountUri, noTestAccountConf);
-
- // Create test account initializing the appropriate member variables.
- AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(fs,
- account, container);
-
- // Return to caller with test account.
- return testAcct;
- }
-
- private static CloudBlockBlob primeRootContainer(CloudBlobClient blobClient,
- String accountName, String blobName, int fileSize) throws Exception {
-
- // Create a container if it does not exist. The container name
- // must be lower case.
- CloudBlobContainer container = blobClient.getContainerReference("https://"
- + accountName + "/" + "$root");
- container.createIfNotExists();
-
- // Create a blob output stream.
- CloudBlockBlob blob = container.getBlockBlobReference(blobName);
- BlobOutputStream outputStream = blob.openOutputStream();
-
- outputStream.write(new byte[fileSize]);
- outputStream.close();
-
- // Return a reference to the block blob object.
- return blob;
- }
-
- public static AzureBlobStorageTestAccount createRoot(final String blobName,
- final int fileSize) throws Exception {
-
- NativeAzureFileSystem fs = null;
- CloudBlobContainer container = null;
- Configuration conf = createTestConfiguration();
-
- // Set up a session with the cloud blob client to generate SAS and check the
- // existence of a container and capture the container object.
- CloudStorageAccount account = createTestAccount(conf);
- if (account == null) {
- return null;
- }
- CloudBlobClient blobClient = account.createCloudBlobClient();
-
- // Capture the account URL and the account name.
- String accountName = verifyWasbAccountNameInConfig(conf);
-
- configureSecureModeTestSettings(conf);
-
- // Set up public container with the specified blob name.
- CloudBlockBlob blobRoot = primeRootContainer(blobClient, accountName,
- blobName, fileSize);
-
- // Capture the blob container object. It should exist after generating the
- // shared access signature.
- container = blobClient.getContainerReference(AZURE_ROOT_CONTAINER);
- if (null == container || !container.exists()) {
- final String errMsg = String
- .format("Container '%s' expected but not found while creating SAS account.");
- throw new Exception(errMsg);
- }
-
- // Set the account URI without a container name.
- URI accountUri = createAccountUri(accountName);
-
- // Initialize the Native Azure file system with anonymous credentials.
- fs = new NativeAzureFileSystem();
- fs.initialize(accountUri, conf);
-
- // Create test account initializing the appropriate member variables.
- // Set the container value to null for the default root container.
- //
- AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount(
- fs, account, blobRoot);
-
- // Return to caller with test account.
- return testAcct;
- }
-
- public void closeFileSystem() throws Exception {
- if (fs != null) {
- fs.close();
- }
- }
-
- public void cleanup() throws Exception {
- if (fs != null) {
- fs.close();
- fs = null;
- }
- if (!skipContainerDelete && container != null) {
- container.deleteIfExists();
- container = null;
- }
- if (blob != null) {
- // The blob member variable is set for blobs under root containers.
- // Delete blob objects created for root container tests when cleaning
- // up the test account.
- blob.delete();
- blob = null;
- }
- }
-
- @Override
- public void close() throws Exception {
- cleanup();
- }
-
- public NativeAzureFileSystem getFileSystem() {
- return fs;
- }
-
- public AzureNativeFileSystemStore getStore() {
- return this.storage;
- }
-
- /**
- * Gets the real blob container backing this account if it's not a mock.
- *
- * @return A container, or null if it's a mock.
- */
- public CloudBlobContainer getRealContainer() {
- return container;
- }
-
- /**
- * Gets the real blob account backing this account if it's not a mock.
- *
- * @return An account, or null if it's a mock.
- */
- public CloudStorageAccount getRealAccount() {
- return account;
- }
-
- /**
- * Gets the mock storage interface if this account is backed by a mock.
- *
- * @return The mock storage, or null if it's backed by a real account.
- */
- public MockStorageInterface getMockStorage() {
- return mockStorage;
- }
-
- public static class StandardCollector implements MetricsSink {
- @Override
- public void init(SubsetConfiguration conf) {
- }
-
- @Override
- public void putMetrics(MetricsRecord record) {
- addRecord(record);
- }
-
- @Override
- public void flush() {
- }
- }
-
- public void setPageBlobDirectory(String directory) {
- this.pageBlobDirectory = directory;
- }
-
- public String getPageBlobDirectory() {
- return pageBlobDirectory;
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ExceptionHandlingTestHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ExceptionHandlingTestHelper.java
deleted file mode 100644
index bea1c76d48e0d..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ExceptionHandlingTestHelper.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.conf.Configuration;
-
-public class ExceptionHandlingTestHelper {
-
- /*
- * Helper method to create a PageBlob test storage account.
- */
- public static AzureBlobStorageTestAccount getPageBlobTestStorageAccount()
- throws Exception {
-
- Configuration conf = new Configuration();
-
- // Configure the page blob directories key so every file created is a page blob.
- conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, "/");
-
- // Configure the atomic rename directories key so every folder will have
- // atomic rename applied.
- conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES, "/");
- return AzureBlobStorageTestAccount.create(conf);
- }
-
- /*
- * Helper method to create an empty file
- */
- public static void createEmptyFile(AzureBlobStorageTestAccount testAccount, Path testPath) throws Exception {
- FileSystem fs = testAccount.getFileSystem();
- FSDataOutputStream inputStream = fs.create(testPath);
- inputStream.close();
- }
-
- /*
- * Helper method to create an folder and files inside it.
- */
- public static void createTestFolder(AzureBlobStorageTestAccount testAccount, Path testFolderPath) throws Exception {
- FileSystem fs = testAccount.getFileSystem();
- fs.mkdirs(testFolderPath);
- String testFolderFilePathBase = "test";
-
- for (int i = 0; i < 10; i++) {
- Path p = new Path(testFolderPath.toString() + "/" + testFolderFilePathBase + i + ".dat");
- fs.create(p).close();
- }
- }
-}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
deleted file mode 100644
index c008d64386bf8..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Arrays;
-
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
-
-/**
- * Handle OOB IO into a shared container.
- */
-public class ITestAzureConcurrentOutOfBandIo extends AbstractWasbTestBase {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ITestAzureConcurrentOutOfBandIo.class);
-
- // Class constants.
- static final int DOWNLOAD_BLOCK_SIZE = 8 * 1024 * 1024;
- static final int UPLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
- static final int BLOB_SIZE = 32 * 1024 * 1024;
-
- // Number of blocks to be written before flush.
- static final int NUMBER_OF_BLOCKS = 2;
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.createOutOfBandStore(
- UPLOAD_BLOCK_SIZE, DOWNLOAD_BLOCK_SIZE);
- }
-
- class DataBlockWriter implements Runnable {
-
- Thread runner;
- AzureBlobStorageTestAccount writerStorageAccount;
- String key;
- boolean done = false;
-
- /**
- * Constructor captures the test account.
- *
- * @param testAccount
- */
- public DataBlockWriter(AzureBlobStorageTestAccount testAccount, String key) {
- writerStorageAccount = testAccount;
- this.key = key;
- }
-
- /**
- * Start writing blocks to Azure storage.
- */
- public void startWriting() {
- runner = new SubjectInheritingThread(this); // Create the block writer thread.
- runner.start(); // Start the block writer thread.
- }
-
- /**
- * Stop writing blocks to Azure storage.
- */
- public void stopWriting() {
- done = true;
- }
-
- /**
- * Implementation of the runnable interface. The run method is a tight loop
- * which repeatedly updates the blob with a 4 MB block.
- */
- public void run() {
- byte[] dataBlockWrite = new byte[UPLOAD_BLOCK_SIZE];
-
- OutputStream outputStream = null;
-
- try {
- for (int i = 0; !done; i++) {
- // Write two 4 MB blocks to the blob.
- //
- outputStream = writerStorageAccount.getStore().storefile(
- key,
- new PermissionStatus("", "", FsPermission.getDefault()),
- key);
-
- Arrays.fill(dataBlockWrite, (byte) (i % 256));
- for (int j = 0; j < NUMBER_OF_BLOCKS; j++) {
- outputStream.write(dataBlockWrite);
- }
-
- outputStream.flush();
- outputStream.close();
- }
- } catch (AzureException e) {
- LOG.error("DatablockWriter thread encountered a storage exception."
- + e.getMessage(), e);
- } catch (IOException e) {
- LOG.error("DatablockWriter thread encountered an I/O exception."
- + e.getMessage(), e);
- }
- }
- }
-
- @Test
- public void testReadOOBWrites() throws Exception {
-
- byte[] dataBlockWrite = new byte[UPLOAD_BLOCK_SIZE];
- byte[] dataBlockRead = new byte[UPLOAD_BLOCK_SIZE];
-
- // Write to blob to make sure it exists.
- //
- // Write five 4 MB blocks to the blob. To ensure there is data in the blob before
- // reading. This eliminates the race between the reader and writer threads.
- String key = "WASB_String" + AzureTestUtils.getForkID() + ".txt";
- OutputStream outputStream = testAccount.getStore().storefile(
- key,
- new PermissionStatus("", "", FsPermission.getDefault()),
- key);
- Arrays.fill(dataBlockWrite, (byte) 255);
- for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
- outputStream.write(dataBlockWrite);
- }
-
- outputStream.flush();
- outputStream.close();
-
- // Start writing blocks to Azure store using the DataBlockWriter thread.
- DataBlockWriter writeBlockTask = new DataBlockWriter(testAccount, key);
- writeBlockTask.startWriting();
- int count = 0;
-
- for (int i = 0; i < 5; i++) {
- try(InputStream inputStream = testAccount.getStore().retrieve(key)) {
- count = 0;
- int c = 0;
-
- while (c >= 0) {
- c = inputStream.read(dataBlockRead, 0, UPLOAD_BLOCK_SIZE);
- if (c < 0) {
- break;
- }
-
- // Counting the number of bytes.
- count += c;
- }
- } catch (IOException e) {
- System.out.println(e.getCause().toString());
- e.printStackTrace();
- fail();
- }
- }
-
- // Stop writing blocks.
- writeBlockTask.stopWriting();
-
- // Validate that a block was read.
- assertEquals(NUMBER_OF_BLOCKS * UPLOAD_BLOCK_SIZE, count);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
deleted file mode 100644
index 2b0ea56821c9e..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-/**
- * Extends ITestAzureConcurrentOutOfBandIo in order to run testReadOOBWrites with secure mode
- * (fs.azure.secure.mode) both enabled and disabled.
- */
-public class ITestAzureConcurrentOutOfBandIoWithSecureMode
- extends ITestAzureConcurrentOutOfBandIo {
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.createOutOfBandStore(
- UPLOAD_BLOCK_SIZE, DOWNLOAD_BLOCK_SIZE, true);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
deleted file mode 100644
index a4baab22dad04..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.concurrent.Callable;
-
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.SendingRequestEvent;
-import com.microsoft.azure.storage.StorageEvent;
-import org.junit.jupiter.api.Test;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-/**
- * Error handling.
- */
-public class ITestAzureFileSystemErrorConditions extends
- AbstractWasbTestWithTimeout {
- private static final int ALL_THREE_FILE_SIZE = 1024;
-
- @Test
- public void testNoInitialize() throws Exception {
- intercept(AssertionError.class,
- new Callable() {
- @Override
- public FileMetadata call() throws Exception {
- return new AzureNativeFileSystemStore()
- .retrieveMetadata("foo");
- }
- });
- }
-
- /**
- * Try accessing an unauthorized or non-existent (treated the same) container
- * from WASB.
- */
- @Test
- public void testAccessUnauthorizedPublicContainer() throws Exception {
- final String container = "nonExistentContainer";
- final String account = "hopefullyNonExistentAccount";
- Path noAccessPath = new Path(
- "wasb://" + container + "@" + account + "/someFile");
- NativeAzureFileSystem.suppressRetryPolicy();
- try {
- FileSystem.get(noAccessPath.toUri(), new Configuration())
- .open(noAccessPath);
- assertTrue(false, "Should've thrown.");
- } catch (AzureException ex) {
- GenericTestUtils.assertExceptionContains(
- String.format(NO_ACCESS_TO_CONTAINER_MSG, account, container), ex);
- } finally {
- NativeAzureFileSystem.resumeRetryPolicy();
- }
- }
-
- @Test
- public void testAccessContainerWithWrongVersion() throws Exception {
- AzureNativeFileSystemStore store = new AzureNativeFileSystemStore();
- MockStorageInterface mockStorage = new MockStorageInterface();
- store.setAzureStorageInteractionLayer(mockStorage);
- try (FileSystem fs = new NativeAzureFileSystem(store)) {
- Configuration conf = new Configuration();
- AzureBlobStorageTestAccount.setMockAccountKey(conf);
- HashMap metadata = new HashMap();
- metadata.put(AzureNativeFileSystemStore.VERSION_METADATA_KEY,
- "2090-04-05"); // It's from the future!
- mockStorage.addPreExistingContainer(
- AzureBlobStorageTestAccount.getMockContainerUri(), metadata);
-
- AzureException ex = intercept(AzureException.class,
- new Callable() {
- @Override
- public FileStatus[] call() throws Exception {
- fs.initialize(new URI(AzureBlobStorageTestAccount.MOCK_WASB_URI),
- conf);
- return fs.listStatus(new Path("/"));
- }
- });
- GenericTestUtils.assertExceptionContains(
- "unsupported version: 2090-04-05.", ex);
- }
- }
-
- private interface ConnectionRecognizer {
- boolean isTargetConnection(HttpURLConnection connection);
- }
-
- private class TransientErrorInjector extends StorageEvent {
- private final ConnectionRecognizer connectionRecognizer;
- private boolean injectedErrorOnce = false;
-
- public TransientErrorInjector(ConnectionRecognizer connectionRecognizer) {
- this.connectionRecognizer = connectionRecognizer;
- }
-
- @Override
- public void eventOccurred(SendingRequestEvent eventArg) {
- HttpURLConnection connection
- = (HttpURLConnection) eventArg.getConnectionObject();
- if (!connectionRecognizer.isTargetConnection(connection)) {
- return;
- }
- if (!injectedErrorOnce) {
- connection.setReadTimeout(1);
- connection.disconnect();
- injectedErrorOnce = true;
- }
- }
- }
-
- private void injectTransientError(NativeAzureFileSystem fs,
- final ConnectionRecognizer connectionRecognizer) {
- fs.getStore().addTestHookToOperationContext(new TestHookOperationContext() {
- @Override
- public OperationContext modifyOperationContext(OperationContext original) {
- original.getSendingRequestEventHandler().addListener(
- new TransientErrorInjector(connectionRecognizer));
- return original;
- }
- });
- }
-
- @Test
- public void testTransientErrorOnDelete() throws Exception {
- // Need to do this test against a live storage account
- AzureBlobStorageTestAccount testAccount =
- AzureBlobStorageTestAccount.create();
- assumeNotNull(testAccount);
- try {
- NativeAzureFileSystem fs = testAccount.getFileSystem();
- injectTransientError(fs, new ConnectionRecognizer() {
- @Override
- public boolean isTargetConnection(HttpURLConnection connection) {
- return connection.getRequestMethod().equals("DELETE");
- }
- });
- Path testFile = new Path("/a/b");
- assertTrue(fs.createNewFile(testFile));
- assertTrue(fs.rename(testFile, new Path("/x")));
- } finally {
- testAccount.cleanup();
- }
- }
-
- private void writeAllThreeFile(NativeAzureFileSystem fs, Path testFile)
- throws IOException {
- byte[] buffer = new byte[ALL_THREE_FILE_SIZE];
- Arrays.fill(buffer, (byte) 3);
- try(OutputStream stream = fs.create(testFile)) {
- stream.write(buffer);
- }
- }
-
- private void readAllThreeFile(NativeAzureFileSystem fs, Path testFile)
- throws IOException {
- byte[] buffer = new byte[ALL_THREE_FILE_SIZE];
- InputStream inStream = fs.open(testFile);
- assertEquals(buffer.length,
- inStream.read(buffer, 0, buffer.length));
- inStream.close();
- for (int i = 0; i < buffer.length; i++) {
- assertEquals(3, buffer[i]);
- }
- }
-
- @Test
- public void testTransientErrorOnCommitBlockList() throws Exception {
- // Need to do this test against a live storage account
- AzureBlobStorageTestAccount testAccount =
- AzureBlobStorageTestAccount.create();
- assumeNotNull(testAccount);
- try {
- NativeAzureFileSystem fs = testAccount.getFileSystem();
- injectTransientError(fs, new ConnectionRecognizer() {
- @Override
- public boolean isTargetConnection(HttpURLConnection connection) {
- return connection.getRequestMethod().equals("PUT")
- && connection.getURL().getQuery() != null
- && connection.getURL().getQuery().contains("blocklist");
- }
- });
- Path testFile = new Path("/a/b");
- writeAllThreeFile(fs, testFile);
- readAllThreeFile(fs, testFile);
- } finally {
- testAccount.cleanup();
- }
- }
-
- @Test
- public void testTransientErrorOnRead() throws Exception {
- // Need to do this test against a live storage account
- AzureBlobStorageTestAccount testAccount =
- AzureBlobStorageTestAccount.create();
- assumeNotNull(testAccount);
- try {
- NativeAzureFileSystem fs = testAccount.getFileSystem();
- Path testFile = new Path("/a/b");
- writeAllThreeFile(fs, testFile);
- injectTransientError(fs, new ConnectionRecognizer() {
- @Override
- public boolean isTargetConnection(HttpURLConnection connection) {
- return connection.getRequestMethod().equals("GET");
- }
- });
- readAllThreeFile(fs, testFile);
- } finally {
- testAccount.cleanup();
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
deleted file mode 100644
index 6f266c418a414..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_CHECK_BLOCK_MD5;
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_STORE_BLOB_MD5;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.util.Arrays;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Test;
-
-import com.microsoft.azure.storage.Constants;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.ResponseReceivedEvent;
-import com.microsoft.azure.storage.StorageErrorCodeStrings;
-import com.microsoft.azure.storage.StorageEvent;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlockEntry;
-import com.microsoft.azure.storage.blob.BlockSearchMode;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import com.microsoft.azure.storage.core.Base64;
-
-/**
- * Test that we do proper data integrity validation with MD5 checks as
- * configured.
- */
-public class ITestBlobDataValidation extends AbstractWasbTestWithTimeout {
- private AzureBlobStorageTestAccount testAccount;
-
- @AfterEach
- public void tearDown() throws Exception {
- testAccount = AzureTestUtils.cleanupTestAccount(testAccount);
- }
-
- /**
- * Test that by default we don't store the blob-level MD5.
- */
- @Test
- public void testBlobMd5StoreOffByDefault() throws Exception {
- testAccount = AzureBlobStorageTestAccount.create();
- testStoreBlobMd5(false);
- }
-
- /**
- * Test that we get blob-level MD5 storage and validation if we specify that
- * in the configuration.
- */
- @Test
- public void testStoreBlobMd5() throws Exception {
- Configuration conf = new Configuration();
- conf.setBoolean(KEY_STORE_BLOB_MD5, true);
- testAccount = AzureBlobStorageTestAccount.create(conf);
- testStoreBlobMd5(true);
- }
-
- /**
- * Trims a suffix/prefix from the given string. For example if
- * s is given as "/xy" and toTrim is "/", this method returns "xy"
- */
- private static String trim(String s, String toTrim) {
- return StringUtils.removeEnd(StringUtils.removeStart(s, toTrim),
- toTrim);
- }
-
- private void testStoreBlobMd5(boolean expectMd5Stored) throws Exception {
- assumeNotNull(testAccount);
- // Write a test file.
- NativeAzureFileSystem fs = testAccount.getFileSystem();
- Path testFilePath = AzureTestUtils.pathForTests(fs, methodName.getMethodName());
- String testFileKey = trim(testFilePath.toUri().getPath(), "/");
- OutputStream outStream = fs.create(testFilePath);
- outStream.write(new byte[] { 5, 15 });
- outStream.close();
-
- // Check that we stored/didn't store the MD5 field as configured.
- CloudBlockBlob blob = testAccount.getBlobReference(testFileKey);
- blob.downloadAttributes();
- String obtainedMd5 = blob.getProperties().getContentMD5();
- if (expectMd5Stored) {
- assertNotNull(obtainedMd5);
- } else {
- assertNull(obtainedMd5, "Expected no MD5, found: " + obtainedMd5);
- }
-
- // Mess with the content so it doesn't match the MD5.
- String newBlockId = Base64.encode(new byte[] { 55, 44, 33, 22 });
- blob.uploadBlock(newBlockId,
- new ByteArrayInputStream(new byte[] { 6, 45 }), 2);
- blob.commitBlockList(Arrays.asList(new BlockEntry[] { new BlockEntry(
- newBlockId, BlockSearchMode.UNCOMMITTED) }));
-
- // Now read back the content. If we stored the MD5 for the blob content
- // we should get a data corruption error.
- InputStream inStream = fs.open(testFilePath);
- try {
- byte[] inBuf = new byte[100];
- while (inStream.read(inBuf) > 0){
- //nothing;
- }
- inStream.close();
- if (expectMd5Stored) {
- fail("Should've thrown because of data corruption.");
- }
- } catch (IOException ex) {
- if (!expectMd5Stored) {
- throw ex;
- }
- StorageException cause = (StorageException)ex.getCause();
- assertNotNull(cause);
- assertEquals(StorageErrorCodeStrings.INVALID_MD5, cause.getErrorCode(),
- "Unexpected cause: " + cause);
- }
- }
-
- /**
- * Test that by default we check block-level MD5.
- */
- @Test
- public void testCheckBlockMd5() throws Exception {
- testAccount = AzureBlobStorageTestAccount.create();
- testCheckBlockMd5(true);
- }
-
- /**
- * Test that we don't check block-level MD5 if we specify that in the
- * configuration.
- */
- @Test
- public void testDontCheckBlockMd5() throws Exception {
- Configuration conf = new Configuration();
- conf.setBoolean(KEY_CHECK_BLOCK_MD5, false);
- testAccount = AzureBlobStorageTestAccount.create(conf);
- testCheckBlockMd5(false);
- }
-
- /**
- * Connection inspector to check that MD5 fields for content is set/not set as
- * expected.
- */
- private static class ContentMD5Checker extends
- StorageEvent {
- private final boolean expectMd5;
-
- public ContentMD5Checker(boolean expectMd5) {
- this.expectMd5 = expectMd5;
- }
-
- @Override
- public void eventOccurred(ResponseReceivedEvent eventArg) {
- HttpURLConnection connection = (HttpURLConnection) eventArg
- .getConnectionObject();
- if (isGetRange(connection)) {
- checkObtainedMd5(connection
- .getHeaderField(Constants.HeaderConstants.CONTENT_MD5));
- } else if (isPutBlock(connection)) {
- checkObtainedMd5(connection
- .getRequestProperty(Constants.HeaderConstants.CONTENT_MD5));
- }
- }
-
- private void checkObtainedMd5(String obtainedMd5) {
- if (expectMd5) {
- assertNotNull(obtainedMd5);
- } else {
- assertNull(obtainedMd5, "Expected no MD5, found: " + obtainedMd5);
- }
- }
-
- private static boolean isPutBlock(HttpURLConnection connection) {
- return connection.getRequestMethod().equals("PUT")
- && connection.getURL().getQuery() != null
- && connection.getURL().getQuery().contains("blockid");
- }
-
- private static boolean isGetRange(HttpURLConnection connection) {
- return connection.getRequestMethod().equals("GET")
- && connection
- .getHeaderField(Constants.HeaderConstants.STORAGE_RANGE_HEADER) != null;
- }
- }
-
- private void testCheckBlockMd5(final boolean expectMd5Checked)
- throws Exception {
- assumeNotNull(testAccount);
- Path testFilePath = new Path("/testFile");
-
- // Add a hook to check that for GET/PUT requests we set/don't set
- // the block-level MD5 field as configured. I tried to do clever
- // testing by also messing with the raw data to see if we actually
- // validate the data as expected, but the HttpURLConnection wasn't
- // pluggable enough for me to do that.
- testAccount.getFileSystem().getStore()
- .addTestHookToOperationContext(new TestHookOperationContext() {
- @Override
- public OperationContext modifyOperationContext(
- OperationContext original) {
- original.getResponseReceivedEventHandler().addListener(
- new ContentMD5Checker(expectMd5Checked));
- return original;
- }
- });
-
- OutputStream outStream = testAccount.getFileSystem().create(testFilePath);
- outStream.write(new byte[] { 5, 15 });
- outStream.close();
-
- InputStream inStream = testAccount.getFileSystem().open(testFilePath);
- byte[] inBuf = new byte[100];
- while (inStream.read(inBuf) > 0){
- //nothing;
- }
- inStream.close();
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobTypeSpeedDifference.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobTypeSpeedDifference.java
deleted file mode 100644
index 5c3f156e304c2..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobTypeSpeedDifference.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.Date;
-
-import org.junit.jupiter.api.Test;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
-
-
-/**
- * A simple benchmark to find out the difference in speed between block
- * and page blobs.
- */
-public class ITestBlobTypeSpeedDifference extends AbstractWasbTestBase {
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-
- /**
- * Writes data to the given stream of the given size, flushing every
- * x bytes.
- */
- private static void writeTestFile(OutputStream writeStream,
- long size, long flushInterval) throws IOException {
- int bufferSize = (int) Math.min(1000, flushInterval);
- byte[] buffer = new byte[bufferSize];
- Arrays.fill(buffer, (byte) 7);
- int bytesWritten = 0;
- int bytesUnflushed = 0;
- while (bytesWritten < size) {
- int numberToWrite = (int) Math.min(bufferSize, size - bytesWritten);
- writeStream.write(buffer, 0, numberToWrite);
- bytesWritten += numberToWrite;
- bytesUnflushed += numberToWrite;
- if (bytesUnflushed >= flushInterval) {
- writeStream.flush();
- bytesUnflushed = 0;
- }
- }
- }
-
- private static class TestResult {
- final long timeTakenInMs;
- final long totalNumberOfRequests;
-
- TestResult(long timeTakenInMs, long totalNumberOfRequests) {
- this.timeTakenInMs = timeTakenInMs;
- this.totalNumberOfRequests = totalNumberOfRequests;
- }
- }
-
- /**
- * Writes data to the given file of the given size, flushing every
- * x bytes. Measure performance of that and return it.
- */
- private static TestResult writeTestFile(NativeAzureFileSystem fs, Path path,
- long size, long flushInterval) throws IOException {
- AzureFileSystemInstrumentation instrumentation =
- fs.getInstrumentation();
- long initialRequests = instrumentation.getCurrentWebResponses();
- Date start = new Date();
- OutputStream output = fs.create(path);
- writeTestFile(output, size, flushInterval);
- output.close();
- long finalRequests = instrumentation.getCurrentWebResponses();
- return new TestResult(new Date().getTime() - start.getTime(),
- finalRequests - initialRequests);
- }
-
- /**
- * Writes data to a block blob of the given size, flushing every
- * x bytes. Measure performance of that and return it.
- */
- private static TestResult writeBlockBlobTestFile(NativeAzureFileSystem fs,
- long size, long flushInterval) throws IOException {
- return writeTestFile(fs, new Path("/blockBlob"), size, flushInterval);
- }
-
- /**
- * Writes data to a page blob of the given size, flushing every
- * x bytes. Measure performance of that and return it.
- */
- private static TestResult writePageBlobTestFile(NativeAzureFileSystem fs,
- long size, long flushInterval) throws IOException {
- Path testFile = AzureTestUtils.blobPathForTests(fs,
- "writePageBlobTestFile");
- return writeTestFile(fs,
- testFile,
- size, flushInterval);
- }
-
- /**
- * Runs the benchmark over a small 10 KB file, flushing every 500 bytes.
- */
- @Test
- public void testTenKbFileFrequentFlush() throws Exception {
- testForSizeAndFlushInterval(getFileSystem(), 10 * 1000, 500);
- }
-
- /**
- * Runs the benchmark for the given file size and flush frequency.
- */
- private static void testForSizeAndFlushInterval(NativeAzureFileSystem fs,
- final long size, final long flushInterval) throws IOException {
- for (int i = 0; i < 5; i++) {
- TestResult pageBlobResults = writePageBlobTestFile(fs, size, flushInterval);
- System.out.printf(
- "Page blob upload took %d ms. Total number of requests: %d.\n",
- pageBlobResults.timeTakenInMs, pageBlobResults.totalNumberOfRequests);
- TestResult blockBlobResults = writeBlockBlobTestFile(fs, size, flushInterval);
- System.out.printf(
- "Block blob upload took %d ms. Total number of requests: %d.\n",
- blockBlobResults.timeTakenInMs, blockBlobResults.totalNumberOfRequests);
- }
- }
-
- /**
- * Runs the benchmark for the given file size and flush frequency from the
- * command line.
- */
- public static void main(String[] argv) throws Exception {
- Configuration conf = new Configuration();
- long size = 10 * 1000 * 1000;
- long flushInterval = 2000;
- if (argv.length > 0) {
- size = Long.parseLong(argv[0]);
- }
- if (argv.length > 1) {
- flushInterval = Long.parseLong(argv[1]);
- }
- testForSizeAndFlushInterval(
- (NativeAzureFileSystem) FileSystem.get(conf),
- size,
- flushInterval);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlockBlobInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlockBlobInputStream.java
deleted file mode 100644
index 778bbb8849b70..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlockBlobInputStream.java
+++ /dev/null
@@ -1,927 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Random;
-import java.util.concurrent.Callable;
-
-import org.junit.jupiter.api.MethodOrderer;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestMethodOrder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.integration.AbstractAzureScaleTest;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.contract.ContractTestUtils.NanoTimer;
-
-import static org.apache.hadoop.test.LambdaTestUtils.*;
-
-/**
- * Test semantics and performance of the original block blob input stream
- * (KEY_INPUT_STREAM_VERSION=1) and the new
- * BlockBlobInputStream (KEY_INPUT_STREAM_VERSION=2).
- */
-@TestMethodOrder(MethodOrderer.Alphanumeric.class)
-
-public class ITestBlockBlobInputStream extends AbstractAzureScaleTest {
- private static final Logger LOG = LoggerFactory.getLogger(
- ITestBlockBlobInputStream.class);
- private static final int KILOBYTE = 1024;
- private static final int MEGABYTE = KILOBYTE * KILOBYTE;
- private static final int TEST_FILE_SIZE = 6 * MEGABYTE;
- private static final Path TEST_FILE_PATH = new Path(
- "TestBlockBlobInputStream.txt");
-
- private AzureBlobStorageTestAccount accountUsingInputStreamV1;
- private AzureBlobStorageTestAccount accountUsingInputStreamV2;
- private long testFileLength;
-
-
-
- private FileStatus testFileStatus;
- private Path hugefile;
-
- @BeforeEach
- @Override
- public void setUp() throws Exception {
- super.setUp();
- Configuration conf = new Configuration();
- conf.setInt(AzureNativeFileSystemStore.KEY_INPUT_STREAM_VERSION, 1);
-
- accountUsingInputStreamV1 = AzureBlobStorageTestAccount.create(
- "testblockblobinputstream",
- EnumSet.of(AzureBlobStorageTestAccount.CreateOptions.CreateContainer),
- conf,
- true);
-
- accountUsingInputStreamV2 = AzureBlobStorageTestAccount.create(
- "testblockblobinputstream",
- EnumSet.noneOf(AzureBlobStorageTestAccount.CreateOptions.class),
- null,
- true);
-
- assumeNotNull(accountUsingInputStreamV1);
- assumeNotNull(accountUsingInputStreamV2);
- hugefile = fs.makeQualified(TEST_FILE_PATH);
- try {
- testFileStatus = fs.getFileStatus(TEST_FILE_PATH);
- testFileLength = testFileStatus.getLen();
- } catch (FileNotFoundException e) {
- // file doesn't exist
- testFileLength = 0;
- }
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- Configuration conf = new Configuration();
- conf.setInt(AzureNativeFileSystemStore.KEY_INPUT_STREAM_VERSION, 1);
-
- accountUsingInputStreamV1 = AzureBlobStorageTestAccount.create(
- "testblockblobinputstream",
- EnumSet.of(AzureBlobStorageTestAccount.CreateOptions.CreateContainer),
- conf,
- true);
-
- accountUsingInputStreamV2 = AzureBlobStorageTestAccount.create(
- "testblockblobinputstream",
- EnumSet.noneOf(AzureBlobStorageTestAccount.CreateOptions.class),
- null,
- true);
-
- assumeNotNull(accountUsingInputStreamV1);
- assumeNotNull(accountUsingInputStreamV2);
- return accountUsingInputStreamV1;
- }
-
- /**
- * Create a test file by repeating the characters in the alphabet.
- * @throws IOException
- */
- private void createTestFileAndSetLength() throws IOException {
- FileSystem fs = accountUsingInputStreamV1.getFileSystem();
-
- // To reduce test run time, the test file can be reused.
- if (fs.exists(TEST_FILE_PATH)) {
- testFileStatus = fs.getFileStatus(TEST_FILE_PATH);
- testFileLength = testFileStatus.getLen();
- LOG.info("Reusing test file: {}", testFileStatus);
- return;
- }
-
- int sizeOfAlphabet = ('z' - 'a' + 1);
- byte[] buffer = new byte[26 * KILOBYTE];
- char character = 'a';
- for (int i = 0; i < buffer.length; i++) {
- buffer[i] = (byte) character;
- character = (character == 'z') ? 'a' : (char) ((int) character + 1);
- }
-
- LOG.info("Creating test file {} of size: {}", TEST_FILE_PATH,
- TEST_FILE_SIZE);
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
-
- try(FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH)) {
- int bytesWritten = 0;
- while (bytesWritten < TEST_FILE_SIZE) {
- outputStream.write(buffer);
- bytesWritten += buffer.length;
- }
- LOG.info("Closing stream {}", outputStream);
- ContractTestUtils.NanoTimer closeTimer
- = new ContractTestUtils.NanoTimer();
- outputStream.close();
- closeTimer.end("time to close() output stream");
- }
- timer.end("time to write %d KB", TEST_FILE_SIZE / 1024);
- testFileLength = fs.getFileStatus(TEST_FILE_PATH).getLen();
- }
-
- void assumeHugeFileExists() throws IOException {
- ContractTestUtils.assertPathExists(fs, "huge file not created", hugefile);
- FileStatus status = fs.getFileStatus(hugefile);
- ContractTestUtils.assertIsFile(hugefile, status);
- assertTrue(status.getLen() > 0, "File " + hugefile + " is empty");
- }
-
- /**
- * Calculate megabits per second from the specified values for bytes and
- * milliseconds.
- * @param bytes The number of bytes.
- * @param milliseconds The number of milliseconds.
- * @return The number of megabits per second.
- */
- private static double toMbps(long bytes, long milliseconds) {
- return bytes / 1000.0 * 8 / milliseconds;
- }
-
- @Test
- public void test_0100_CreateHugeFile() throws IOException {
- createTestFileAndSetLength();
- }
-
- @Test
- public void test_0200_BasicReadTest() throws Exception {
- assumeHugeFileExists();
-
- try (
- FSDataInputStream inputStreamV1
- = accountUsingInputStreamV1.getFileSystem().open(TEST_FILE_PATH);
-
- FSDataInputStream inputStreamV2
- = accountUsingInputStreamV2.getFileSystem().open(TEST_FILE_PATH);
- ) {
- byte[] bufferV1 = new byte[3 * MEGABYTE];
- byte[] bufferV2 = new byte[bufferV1.length];
-
- // v1 forward seek and read a kilobyte into first kilobyte of bufferV1
- inputStreamV1.seek(5 * MEGABYTE);
- int numBytesReadV1 = inputStreamV1.read(bufferV1, 0, KILOBYTE);
- assertEquals(KILOBYTE, numBytesReadV1);
-
- // v2 forward seek and read a kilobyte into first kilobyte of bufferV2
- inputStreamV2.seek(5 * MEGABYTE);
- int numBytesReadV2 = inputStreamV2.read(bufferV2, 0, KILOBYTE);
- assertEquals(KILOBYTE, numBytesReadV2);
-
- assertArrayEquals(bufferV1, bufferV2);
-
- int len = MEGABYTE;
- int offset = bufferV1.length - len;
-
- // v1 reverse seek and read a megabyte into last megabyte of bufferV1
- inputStreamV1.seek(3 * MEGABYTE);
- numBytesReadV1 = inputStreamV1.read(bufferV1, offset, len);
- assertEquals(len, numBytesReadV1);
-
- // v2 reverse seek and read a megabyte into last megabyte of bufferV2
- inputStreamV2.seek(3 * MEGABYTE);
- numBytesReadV2 = inputStreamV2.read(bufferV2, offset, len);
- assertEquals(len, numBytesReadV2);
-
- assertArrayEquals(bufferV1, bufferV2);
- }
- }
-
- @Test
- public void test_0201_RandomReadTest() throws Exception {
- assumeHugeFileExists();
-
- try (
- FSDataInputStream inputStreamV1
- = accountUsingInputStreamV1.getFileSystem().open(TEST_FILE_PATH);
-
- FSDataInputStream inputStreamV2
- = accountUsingInputStreamV2.getFileSystem().open(TEST_FILE_PATH);
- ) {
- final int bufferSize = 4 * KILOBYTE;
- byte[] bufferV1 = new byte[bufferSize];
- byte[] bufferV2 = new byte[bufferV1.length];
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- inputStreamV1.seek(0);
- inputStreamV2.seek(0);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- int seekPosition = 2 * KILOBYTE;
- inputStreamV1.seek(seekPosition);
- inputStreamV2.seek(seekPosition);
-
- inputStreamV1.seek(0);
- inputStreamV2.seek(0);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- seekPosition = 5 * KILOBYTE;
- inputStreamV1.seek(seekPosition);
- inputStreamV2.seek(seekPosition);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- seekPosition = 10 * KILOBYTE;
- inputStreamV1.seek(seekPosition);
- inputStreamV2.seek(seekPosition);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
-
- seekPosition = 4100 * KILOBYTE;
- inputStreamV1.seek(seekPosition);
- inputStreamV2.seek(seekPosition);
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
- }
- }
-
- private void verifyConsistentReads(FSDataInputStream inputStreamV1,
- FSDataInputStream inputStreamV2,
- byte[] bufferV1,
- byte[] bufferV2) throws IOException {
- int size = bufferV1.length;
- final int numBytesReadV1 = inputStreamV1.read(bufferV1, 0, size);
- assertEquals(size, numBytesReadV1, "Bytes read from V1 stream");
-
- final int numBytesReadV2 = inputStreamV2.read(bufferV2, 0, size);
- assertEquals(size, numBytesReadV2, "Bytes read from V2 stream");
-
- assertArrayEquals(bufferV1, bufferV2, "Mismatch in read data");
- }
-
- @Test
- public void test_202_PosReadTest() throws Exception {
- assumeHugeFileExists();
- FutureDataInputStreamBuilder builder = accountUsingInputStreamV2
- .getFileSystem().openFile(TEST_FILE_PATH);
- builder.opt(AzureNativeFileSystemStore.FS_AZURE_BLOCK_BLOB_BUFFERED_PREAD_DISABLE, true);
- try (
- FSDataInputStream inputStreamV1
- = accountUsingInputStreamV1.getFileSystem().open(TEST_FILE_PATH);
- FSDataInputStream inputStreamV2
- = accountUsingInputStreamV2.getFileSystem().open(TEST_FILE_PATH);
- FSDataInputStream inputStreamV2NoBuffer = builder.build().get();
- ) {
- final int bufferSize = 4 * KILOBYTE;
- byte[] bufferV1 = new byte[bufferSize];
- byte[] bufferV2 = new byte[bufferSize];
- byte[] bufferV2NoBuffer = new byte[bufferSize];
-
- verifyConsistentReads(inputStreamV1, inputStreamV2, inputStreamV2NoBuffer, 0,
- bufferV1, bufferV2, bufferV2NoBuffer);
-
- int pos = 2 * KILOBYTE;
- verifyConsistentReads(inputStreamV1, inputStreamV2, inputStreamV2NoBuffer, pos,
- bufferV1, bufferV2, bufferV2NoBuffer);
-
- pos = 10 * KILOBYTE;
- verifyConsistentReads(inputStreamV1, inputStreamV2, inputStreamV2NoBuffer, pos,
- bufferV1, bufferV2, bufferV2NoBuffer);
-
- pos = 4100 * KILOBYTE;
- verifyConsistentReads(inputStreamV1, inputStreamV2, inputStreamV2NoBuffer, pos,
- bufferV1, bufferV2, bufferV2NoBuffer);
- }
- }
-
- private void verifyConsistentReads(FSDataInputStream inputStreamV1,
- FSDataInputStream inputStreamV2, FSDataInputStream inputStreamV2NoBuffer,
- int pos, byte[] bufferV1, byte[] bufferV2, byte[] bufferV2NoBuffer)
- throws IOException {
- int size = bufferV1.length;
- int numBytesReadV1 = inputStreamV1.read(pos, bufferV1, 0, size);
- assertEquals(size, numBytesReadV1, "Bytes read from V1 stream");
-
- int numBytesReadV2 = inputStreamV2.read(pos, bufferV2, 0, size);
- assertEquals(size, numBytesReadV2, "Bytes read from V2 stream");
-
- int numBytesReadV2NoBuffer = inputStreamV2NoBuffer.read(pos,
- bufferV2NoBuffer, 0, size);
- assertEquals(size, numBytesReadV2NoBuffer,
- "Bytes read from V2 stream (buffered pread disabled)");
-
- assertArrayEquals(bufferV1, bufferV2, "Mismatch in read data");
- assertArrayEquals(bufferV2, bufferV2NoBuffer, "Mismatch in read data");
- }
-
- /**
- * Validates the implementation of InputStream.markSupported.
- * @throws IOException
- */
- @Test
- public void test_0301_MarkSupportedV1() throws IOException {
- validateMarkSupported(accountUsingInputStreamV1.getFileSystem());
- }
-
- /**
- * Validates the implementation of InputStream.markSupported.
- * @throws IOException
- */
- @Test
- public void test_0302_MarkSupportedV2() throws IOException {
- validateMarkSupported(accountUsingInputStreamV1.getFileSystem());
- }
-
- private void validateMarkSupported(FileSystem fs) throws IOException {
- assumeHugeFileExists();
- try (FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
- assertTrue(inputStream.markSupported(), "mark is not supported");
- }
- }
-
- /**
- * Validates the implementation of InputStream.mark and reset
- * for version 1 of the block blob input stream.
- * @throws Exception
- */
- @Test
- public void test_0303_MarkAndResetV1() throws Exception {
- validateMarkAndReset(accountUsingInputStreamV1.getFileSystem());
- }
-
- /**
- * Validates the implementation of InputStream.mark and reset
- * for version 2 of the block blob input stream.
- * @throws Exception
- */
- @Test
- public void test_0304_MarkAndResetV2() throws Exception {
- validateMarkAndReset(accountUsingInputStreamV2.getFileSystem());
- }
-
- private void validateMarkAndReset(FileSystem fs) throws Exception {
- assumeHugeFileExists();
- try (FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
- inputStream.mark(KILOBYTE - 1);
-
- byte[] buffer = new byte[KILOBYTE];
- int bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
-
- inputStream.reset();
- assertEquals(0, inputStream.getPos(), "rest -> pos 0");
-
- inputStream.mark(8 * KILOBYTE - 1);
-
- buffer = new byte[8 * KILOBYTE];
- bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
-
- intercept(IOException.class,
- "Resetting to invalid mark",
- new Callable() {
- @Override
- public FSDataInputStream call() throws Exception {
- inputStream.reset();
- return inputStream;
- }
- }
- );
- }
- }
-
- /**
- * Validates the implementation of Seekable.seekToNewSource, which should
- * return false for version 1 of the block blob input stream.
- * @throws IOException
- */
- @Test
- public void test_0305_SeekToNewSourceV1() throws IOException {
- validateSeekToNewSource(accountUsingInputStreamV1.getFileSystem());
- }
-
- /**
- * Validates the implementation of Seekable.seekToNewSource, which should
- * return false for version 2 of the block blob input stream.
- * @throws IOException
- */
- @Test
- public void test_0306_SeekToNewSourceV2() throws IOException {
- validateSeekToNewSource(accountUsingInputStreamV2.getFileSystem());
- }
-
- private void validateSeekToNewSource(FileSystem fs) throws IOException {
- assumeHugeFileExists();
- try (FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
- assertFalse(inputStream.seekToNewSource(0));
- }
- }
-
- /**
- * Validates the implementation of InputStream.skip and ensures there is no
- * network I/O for version 1 of the block blob input stream.
- * @throws Exception
- */
- @Test
- public void test_0307_SkipBoundsV1() throws Exception {
- validateSkipBounds(accountUsingInputStreamV1.getFileSystem());
- }
-
- /**
- * Validates the implementation of InputStream.skip and ensures there is no
- * network I/O for version 2 of the block blob input stream.
- * @throws Exception
- */
- @Test
- public void test_0308_SkipBoundsV2() throws Exception {
- validateSkipBounds(accountUsingInputStreamV2.getFileSystem());
- }
-
- private void validateSkipBounds(FileSystem fs) throws Exception {
- assumeHugeFileExists();
- try (FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
- NanoTimer timer = new NanoTimer();
-
- long skipped = inputStream.skip(-1);
- assertEquals(0, skipped);
-
- skipped = inputStream.skip(0);
- assertEquals(0, skipped);
-
- assertTrue(testFileLength > 0);
-
- skipped = inputStream.skip(testFileLength);
- assertEquals(testFileLength, skipped);
-
- intercept(EOFException.class,
- new Callable() {
- @Override
- public Long call() throws Exception {
- return inputStream.skip(1);
- }
- }
- );
- long elapsedTimeMs = timer.elapsedTimeMs();
- assertTrue(elapsedTimeMs < 20, String.format(
- "There should not be any network I/O (elapsedTimeMs=%1$d).",
- elapsedTimeMs));
- }
- }
-
- /**
- * Validates the implementation of Seekable.seek and ensures there is no
- * network I/O for forward seek.
- * @throws Exception
- */
- @Test
- public void test_0309_SeekBoundsV1() throws Exception {
- validateSeekBounds(accountUsingInputStreamV1.getFileSystem());
- }
-
- /**
- * Validates the implementation of Seekable.seek and ensures there is no
- * network I/O for forward seek.
- * @throws Exception
- */
- @Test
- public void test_0310_SeekBoundsV2() throws Exception {
- validateSeekBounds(accountUsingInputStreamV2.getFileSystem());
- }
-
- private void validateSeekBounds(FileSystem fs) throws Exception {
- assumeHugeFileExists();
- try (
- FSDataInputStream inputStream = fs.open(TEST_FILE_PATH);
- ) {
- NanoTimer timer = new NanoTimer();
-
- inputStream.seek(0);
- assertEquals(0, inputStream.getPos());
-
- intercept(EOFException.class,
- FSExceptionMessages.NEGATIVE_SEEK,
- new Callable() {
- @Override
- public FSDataInputStream call() throws Exception {
- inputStream.seek(-1);
- return inputStream;
- }
- }
- );
-
- assertTrue(testFileLength > 0, "Test file length only " + testFileLength);
- inputStream.seek(testFileLength);
- assertEquals(testFileLength, inputStream.getPos());
-
- intercept(EOFException.class,
- FSExceptionMessages.CANNOT_SEEK_PAST_EOF,
- new Callable() {
- @Override
- public FSDataInputStream call() throws Exception {
- inputStream.seek(testFileLength + 1);
- return inputStream;
- }
- }
- );
-
- long elapsedTimeMs = timer.elapsedTimeMs();
- assertTrue(
- elapsedTimeMs < 20, String.format(
- "There should not be any network I/O (elapsedTimeMs=%1$d).",
- elapsedTimeMs));
- }
- }
-
- /**
- * Validates the implementation of Seekable.seek, Seekable.getPos,
- * and InputStream.available.
- * @throws Exception
- */
- @Test
- public void test_0311_SeekAndAvailableAndPositionV1() throws Exception {
- validateSeekAndAvailableAndPosition(
- accountUsingInputStreamV1.getFileSystem());
- }
-
- /**
- * Validates the implementation of Seekable.seek, Seekable.getPos,
- * and InputStream.available.
- * @throws Exception
- */
- @Test
- public void test_0312_SeekAndAvailableAndPositionV2() throws Exception {
- validateSeekAndAvailableAndPosition(
- accountUsingInputStreamV2.getFileSystem());
- }
-
- private void validateSeekAndAvailableAndPosition(FileSystem fs)
- throws Exception {
- assumeHugeFileExists();
- try (FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
- byte[] expected1 = {(byte) 'a', (byte) 'b', (byte) 'c'};
- byte[] expected2 = {(byte) 'd', (byte) 'e', (byte) 'f'};
- byte[] expected3 = {(byte) 'b', (byte) 'c', (byte) 'd'};
- byte[] expected4 = {(byte) 'g', (byte) 'h', (byte) 'i'};
- byte[] buffer = new byte[3];
-
- int bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
- assertArrayEquals(expected1, buffer);
- assertEquals(buffer.length, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
-
- bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
- assertArrayEquals(expected2, buffer);
- assertEquals(2 * buffer.length, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
-
- // reverse seek
- int seekPos = 0;
- inputStream.seek(seekPos);
-
- bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
- assertArrayEquals(expected1, buffer);
- assertEquals(buffer.length + seekPos, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
-
- // reverse seek
- seekPos = 1;
- inputStream.seek(seekPos);
-
- bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
- assertArrayEquals(expected3, buffer);
- assertEquals(buffer.length + seekPos, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
-
- // forward seek
- seekPos = 6;
- inputStream.seek(seekPos);
-
- bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
- assertArrayEquals(expected4, buffer);
- assertEquals(buffer.length + seekPos, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
- }
- }
-
- /**
- * Validates the implementation of InputStream.skip, Seekable.getPos,
- * and InputStream.available.
- * @throws IOException
- */
- @Test
- public void test_0313_SkipAndAvailableAndPositionV1() throws IOException {
- validateSkipAndAvailableAndPosition(
- accountUsingInputStreamV1.getFileSystem());
- }
-
- /**
- * Validates the implementation of InputStream.skip, Seekable.getPos,
- * and InputStream.available.
- * @throws IOException
- */
- @Test
- public void test_0314_SkipAndAvailableAndPositionV2() throws IOException {
- validateSkipAndAvailableAndPosition(
- accountUsingInputStreamV1.getFileSystem());
- }
-
- private void validateSkipAndAvailableAndPosition(FileSystem fs)
- throws IOException {
- assumeHugeFileExists();
- try (
- FSDataInputStream inputStream = fs.open(TEST_FILE_PATH);
- ) {
- byte[] expected1 = {(byte) 'a', (byte) 'b', (byte) 'c'};
- byte[] expected2 = {(byte) 'd', (byte) 'e', (byte) 'f'};
- byte[] expected3 = {(byte) 'b', (byte) 'c', (byte) 'd'};
- byte[] expected4 = {(byte) 'g', (byte) 'h', (byte) 'i'};
-
- assertEquals(testFileLength, inputStream.available());
- assertEquals(0, inputStream.getPos());
-
- int n = 3;
- long skipped = inputStream.skip(n);
-
- assertEquals(skipped, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
- assertEquals(skipped, n);
-
- byte[] buffer = new byte[3];
- int bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
- assertArrayEquals(expected2, buffer);
- assertEquals(buffer.length + skipped, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
-
- // does skip still work after seek?
- int seekPos = 1;
- inputStream.seek(seekPos);
-
- bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
- assertArrayEquals(expected3, buffer);
- assertEquals(buffer.length + seekPos, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
-
- long currentPosition = inputStream.getPos();
- n = 2;
- skipped = inputStream.skip(n);
-
- assertEquals(currentPosition + skipped, inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
- assertEquals(skipped, n);
-
- bytesRead = inputStream.read(buffer);
- assertEquals(buffer.length, bytesRead);
- assertArrayEquals(expected4, buffer);
- assertEquals(buffer.length + skipped + currentPosition,
- inputStream.getPos());
- assertEquals(testFileLength - inputStream.getPos(),
- inputStream.available());
- }
- }
-
- /**
- * Ensures parity in the performance of sequential read for
- * version 1 and version 2 of the block blob input stream.
- * @throws IOException
- */
- @Test
- public void test_0315_SequentialReadPerformance() throws IOException {
- assumeHugeFileExists();
- final int maxAttempts = 10;
- final double maxAcceptableRatio = 1.01;
- double v1ElapsedMs = 0, v2ElapsedMs = 0;
- double ratio = Double.MAX_VALUE;
- for (int i = 0; i < maxAttempts && ratio >= maxAcceptableRatio; i++) {
- v1ElapsedMs = sequentialRead(1,
- accountUsingInputStreamV1.getFileSystem(), false);
- v2ElapsedMs = sequentialRead(2,
- accountUsingInputStreamV2.getFileSystem(), false);
- ratio = v2ElapsedMs / v1ElapsedMs;
- LOG.info(String.format(
- "v1ElapsedMs=%1$d, v2ElapsedMs=%2$d, ratio=%3$.2f",
- (long) v1ElapsedMs,
- (long) v2ElapsedMs,
- ratio));
- }
- assertTrue(
- ratio < maxAcceptableRatio, String.format(
- "Performance of version 2 is not acceptable: v1ElapsedMs=%1$d,"
- + " v2ElapsedMs=%2$d, ratio=%3$.2f",
- (long) v1ElapsedMs,
- (long) v2ElapsedMs,
- ratio));
- }
-
- /**
- * Ensures parity in the performance of sequential read after reverse seek for
- * version 2 of the block blob input stream.
- * @throws IOException
- */
- @Test
- public void test_0316_SequentialReadAfterReverseSeekPerformanceV2()
- throws IOException {
- assumeHugeFileExists();
- final int maxAttempts = 10;
- final double maxAcceptableRatio = 1.01;
- double beforeSeekElapsedMs = 0, afterSeekElapsedMs = 0;
- double ratio = Double.MAX_VALUE;
- for (int i = 0; i < maxAttempts && ratio >= maxAcceptableRatio; i++) {
- beforeSeekElapsedMs = sequentialRead(2,
- accountUsingInputStreamV2.getFileSystem(), false);
- afterSeekElapsedMs = sequentialRead(2,
- accountUsingInputStreamV2.getFileSystem(), true);
- ratio = afterSeekElapsedMs / beforeSeekElapsedMs;
- LOG.info(String.format(
- "beforeSeekElapsedMs=%1$d, afterSeekElapsedMs=%2$d, ratio=%3$.2f",
- (long) beforeSeekElapsedMs,
- (long) afterSeekElapsedMs,
- ratio));
- }
- assertTrue(
- ratio < maxAcceptableRatio, String.format(
- "Performance of version 2 after reverse seek is not acceptable:"
- + " beforeSeekElapsedMs=%1$d, afterSeekElapsedMs=%2$d,"
- + " ratio=%3$.2f",
- (long) beforeSeekElapsedMs,
- (long) afterSeekElapsedMs,
- ratio));
- }
-
- private long sequentialRead(int version,
- FileSystem fs,
- boolean afterReverseSeek) throws IOException {
- byte[] buffer = new byte[16 * KILOBYTE];
- long totalBytesRead = 0;
- long bytesRead = 0;
-
- try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
- if (afterReverseSeek) {
- while (bytesRead > 0 && totalBytesRead < 4 * MEGABYTE) {
- bytesRead = inputStream.read(buffer);
- totalBytesRead += bytesRead;
- }
- totalBytesRead = 0;
- inputStream.seek(0);
- }
-
- NanoTimer timer = new NanoTimer();
- while ((bytesRead = inputStream.read(buffer)) > 0) {
- totalBytesRead += bytesRead;
- }
- long elapsedTimeMs = timer.elapsedTimeMs();
-
- LOG.info(String.format(
- "v%1$d: bytesRead=%2$d, elapsedMs=%3$d, Mbps=%4$.2f,"
- + " afterReverseSeek=%5$s",
- version,
- totalBytesRead,
- elapsedTimeMs,
- toMbps(totalBytesRead, elapsedTimeMs),
- afterReverseSeek));
-
- assertEquals(testFileLength, totalBytesRead);
- inputStream.close();
- return elapsedTimeMs;
- }
- }
-
- @Test
- public void test_0317_RandomReadPerformance() throws IOException {
- assumeHugeFileExists();
- final int maxAttempts = 10;
- final double maxAcceptableRatio = 0.10;
- double v1ElapsedMs = 0, v2ElapsedMs = 0;
- double ratio = Double.MAX_VALUE;
- for (int i = 0; i < maxAttempts && ratio >= maxAcceptableRatio; i++) {
- v1ElapsedMs = randomRead(1,
- accountUsingInputStreamV1.getFileSystem());
- v2ElapsedMs = randomRead(2,
- accountUsingInputStreamV2.getFileSystem());
- ratio = v2ElapsedMs / v1ElapsedMs;
- LOG.info(String.format(
- "v1ElapsedMs=%1$d, v2ElapsedMs=%2$d, ratio=%3$.2f",
- (long) v1ElapsedMs,
- (long) v2ElapsedMs,
- ratio));
- }
- assertTrue(
- ratio < maxAcceptableRatio, String.format(
- "Performance of version 2 is not acceptable: v1ElapsedMs=%1$d,"
- + " v2ElapsedMs=%2$d, ratio=%3$.2f",
- (long) v1ElapsedMs,
- (long) v2ElapsedMs,
- ratio));
- }
-
- private long randomRead(int version, FileSystem fs) throws IOException {
- assumeHugeFileExists();
- final int minBytesToRead = 2 * MEGABYTE;
- Random random = new Random();
- byte[] buffer = new byte[8 * KILOBYTE];
- long totalBytesRead = 0;
- long bytesRead = 0;
- try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
- NanoTimer timer = new NanoTimer();
-
- do {
- bytesRead = inputStream.read(buffer);
- totalBytesRead += bytesRead;
- inputStream.seek(random.nextInt(
- (int) (testFileLength - buffer.length)));
- } while (bytesRead > 0 && totalBytesRead < minBytesToRead);
-
- long elapsedTimeMs = timer.elapsedTimeMs();
-
- inputStream.close();
-
- LOG.info(String.format(
- "v%1$d: totalBytesRead=%2$d, elapsedTimeMs=%3$d, Mbps=%4$.2f",
- version,
- totalBytesRead,
- elapsedTimeMs,
- toMbps(totalBytesRead, elapsedTimeMs)));
-
- assertTrue(minBytesToRead <= totalBytesRead);
-
- return elapsedTimeMs;
- }
- }
-
- @Test
- public void test_999_DeleteHugeFiles() throws IOException {
- try {
- NanoTimer timer = new NanoTimer();
- NativeAzureFileSystem fs = getFileSystem();
- fs.delete(TEST_FILE_PATH, false);
- timer.end("time to delete %s", TEST_FILE_PATH);
- } finally {
- // clean up the test account
- AzureTestUtils.cleanupTestAccount(accountUsingInputStreamV1);
- }
- }
-
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestContainerChecks.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestContainerChecks.java
deleted file mode 100644
index 8110b6d6ae8e3..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestContainerChecks.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import static org.assertj.core.api.Assumptions.assumeThat;
-
-import java.io.FileNotFoundException;
-import java.util.EnumSet;
-import java.util.concurrent.Callable;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.test.LambdaTestUtils;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import com.microsoft.azure.storage.blob.BlobOutputStream;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-
-/**
- * Tests that WASB creates containers only if needed.
- */
-public class ITestContainerChecks extends AbstractWasbTestWithTimeout {
- private AzureBlobStorageTestAccount testAccount;
- private boolean runningInSASMode = false;
-
- @AfterEach
- public void tearDown() throws Exception {
- testAccount = AzureTestUtils.cleanup(testAccount);
- }
-
- @BeforeEach
- public void setMode() {
- runningInSASMode = AzureBlobStorageTestAccount.createTestConfiguration().
- getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, false);
- }
-
- @Test
- public void testContainerExistAfterDoesNotExist() throws Exception {
- testAccount = blobStorageTestAccount();
- assumeNotNull(testAccount);
- CloudBlobContainer container = testAccount.getRealContainer();
- FileSystem fs = testAccount.getFileSystem();
-
- // Starting off with the container not there
- assertFalse(container.exists());
-
- // A list shouldn't create the container and will set file system store
- // state to DoesNotExist
- try {
- fs.listStatus(new Path("/"));
- assertTrue(false, "Should've thrown.");
- } catch (FileNotFoundException ex) {
- assertTrue(ex.getMessage().contains("is not found"),
- "Unexpected exception: " + ex);
- }
- assertFalse(container.exists());
-
- // Create a container outside of the WASB FileSystem
- container.create();
- // Add a file to the container outside of the WASB FileSystem
- CloudBlockBlob blob = testAccount.getBlobReference("foo");
- BlobOutputStream outputStream = blob.openOutputStream();
- outputStream.write(new byte[10]);
- outputStream.close();
-
- // Make sure the file is visible
- assertTrue(fs.exists(new Path("/foo")));
- assertTrue(container.exists());
- }
-
- protected AzureBlobStorageTestAccount blobStorageTestAccount()
- throws Exception {
- return AzureBlobStorageTestAccount.create("",
- EnumSet.noneOf(CreateOptions.class));
- }
-
- @Test
- public void testContainerCreateAfterDoesNotExist() throws Exception {
- testAccount = blobStorageTestAccount();
- assumeNotNull(testAccount);
- CloudBlobContainer container = testAccount.getRealContainer();
- FileSystem fs = testAccount.getFileSystem();
-
- // Starting off with the container not there
- assertFalse(container.exists());
-
- // A list shouldn't create the container and will set file system store
- // state to DoesNotExist
- try {
- assertNull(fs.listStatus(new Path("/")));
- assertTrue(false, "Should've thrown.");
- } catch (FileNotFoundException ex) {
- assertTrue(ex.getMessage().contains("is not found"),
- "Unexpected exception: " + ex);
- }
- assertFalse(container.exists());
-
- // Write should succeed
- assertTrue(fs.createNewFile(new Path("/foo")));
- assertTrue(container.exists());
- }
-
- @Test
- public void testContainerCreateOnWrite() throws Exception {
- testAccount = blobStorageTestAccount();
- assumeNotNull(testAccount);
- CloudBlobContainer container = testAccount.getRealContainer();
- FileSystem fs = testAccount.getFileSystem();
-
- // Starting off with the container not there
- assertFalse(container.exists());
-
- // A list shouldn't create the container.
- try {
- fs.listStatus(new Path("/"));
- assertTrue(false, "Should've thrown.");
- } catch (FileNotFoundException ex) {
- assertTrue(ex.getMessage().contains("is not found"),
- "Unexpected exception: " + ex);
- }
- assertFalse(container.exists());
-
- // Neither should a read.
- Path foo = new Path("/testContainerCreateOnWrite-foo");
- Path bar = new Path("/testContainerCreateOnWrite-bar");
- LambdaTestUtils.intercept(FileNotFoundException.class,
- new Callable() {
- @Override
- public String call() throws Exception {
- fs.open(foo).close();
- return "Stream to " + foo;
- }
- }
- );
- assertFalse(container.exists());
-
- // Neither should a rename
- assertFalse(fs.rename(foo, bar));
- assertFalse(container.exists());
-
- // But a write should.
- assertTrue(fs.createNewFile(foo));
- assertTrue(container.exists());
- }
-
- @Test
- public void testContainerChecksWithSas() throws Exception {
-
- assumeThat(runningInSASMode).isFalse();
- testAccount = AzureBlobStorageTestAccount.create("",
- EnumSet.of(CreateOptions.UseSas));
- assumeNotNull(testAccount);
- CloudBlobContainer container = testAccount.getRealContainer();
- FileSystem fs = testAccount.getFileSystem();
-
- // The container shouldn't be there
- assertFalse(container.exists());
-
- // A write should just fail
- try {
- fs.createNewFile(new Path("/testContainerChecksWithSas-foo"));
- assertFalse(true, "Should've thrown.");
- } catch (AzureException ex) {
- }
- assertFalse(container.exists());
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionHandling.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionHandling.java
deleted file mode 100644
index d8293e45c813f..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionHandling.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import static org.apache.hadoop.fs.FSExceptionMessages.STREAM_IS_CLOSED;
-import static org.apache.hadoop.fs.azure.ExceptionHandlingTestHelper.*;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-/**
- * Single threaded exception handling.
- */
-public class ITestFileSystemOperationExceptionHandling
- extends AbstractWasbTestBase {
-
- private FSDataInputStream inputStream = null;
-
- private Path testPath;
- private Path testFolderPath;
-
- @BeforeEach
- @Override
- public void setUp() throws Exception {
- super.setUp();
- testPath = path("testfile.dat");
- testFolderPath = path("testfolder");
- }
-
- /**
- * Helper method that creates a InputStream to validate exceptions
- * for various scenarios.
- */
- private void setupInputStreamToTest(AzureBlobStorageTestAccount testAccount)
- throws Exception {
-
- FileSystem fs = testAccount.getFileSystem();
-
- // Step 1: Create a file and write dummy data.
- Path base = methodPath();
- Path testFilePath1 = new Path(base, "test1.dat");
- Path testFilePath2 = new Path(base, "test2.dat");
- FSDataOutputStream outputStream = fs.create(testFilePath1);
- String testString = "This is a test string";
- outputStream.write(testString.getBytes());
- outputStream.close();
-
- // Step 2: Open a read stream on the file.
- inputStream = fs.open(testFilePath1);
-
- // Step 3: Rename the file
- fs.rename(testFilePath1, testFilePath2);
- }
-
- /**
- * Tests a basic single threaded read scenario for Page blobs.
- */
- @Test
- public void testSingleThreadedPageBlobReadScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- AzureBlobStorageTestAccount testAccount = getPageBlobTestStorageAccount();
- setupInputStreamToTest(testAccount);
- byte[] readBuffer = new byte[512];
- inputStream.read(readBuffer);
- });
- }
-
- /**
- * Tests a basic single threaded seek scenario for Page blobs.
- */
- @Test
- public void testSingleThreadedPageBlobSeekScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- AzureBlobStorageTestAccount testAccount = getPageBlobTestStorageAccount();
- setupInputStreamToTest(testAccount);
- inputStream.seek(5);
- });
- }
-
- /**
- * Test a basic single thread seek scenario for Block blobs.
- */
- @Test
- public void testSingleThreadBlockBlobSeekScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- setupInputStreamToTest(testAccount);
- inputStream.seek(5);
- inputStream.read();
- });
- }
-
- /**
- * Tests a basic single threaded read scenario for Block blobs.
- */
- @Test
- public void testSingledThreadBlockBlobReadScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- setupInputStreamToTest(testAccount);
- byte[] readBuffer = new byte[512];
- inputStream.read(readBuffer);
- });
- }
-
- /**
- * Tests basic single threaded setPermission scenario.
- */
- @Test
- public void testSingleThreadedBlockBlobSetPermissionScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(createTestAccount(), testPath);
- fs.delete(testPath, true);
- fs.setPermission(testPath,
- new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- });
- }
-
- /**
- * Tests basic single threaded setPermission scenario.
- */
- @Test
- public void testSingleThreadedPageBlobSetPermissionScenario()
- throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(getPageBlobTestStorageAccount(), testPath);
- fs.delete(testPath, true);
- fs.setOwner(testPath, "testowner", "testgroup");
- });
- }
-
- /**
- * Tests basic single threaded setPermission scenario.
- */
- @Test
- public void testSingleThreadedBlockBlobSetOwnerScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(createTestAccount(), testPath);
- fs.delete(testPath, true);
- fs.setOwner(testPath, "testowner", "testgroup");
- });
- }
-
- /**
- * Tests basic single threaded setPermission scenario.
- */
- @Test
- public void testSingleThreadedPageBlobSetOwnerScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, ()->{
- createEmptyFile(getPageBlobTestStorageAccount(), testPath);
- fs.delete(testPath, true);
- fs.setPermission(testPath,
- new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- });
- }
-
- /**
- * Test basic single threaded listStatus scenario.
- */
- @Test
- public void testSingleThreadedBlockBlobListStatusScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createTestFolder(createTestAccount(), testFolderPath);
- fs.delete(testFolderPath, true);
- fs.listStatus(testFolderPath);
- });
- }
-
- /**
- * Test basic single threaded listStatus scenario.
- */
- @Test
- public void testSingleThreadedPageBlobListStatusScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createTestFolder(getPageBlobTestStorageAccount(), testFolderPath);
- fs.delete(testFolderPath, true);
- fs.listStatus(testFolderPath);
- });
- }
-
- /**
- * Test basic single threaded listStatus scenario.
- */
- @Test
- public void testSingleThreadedBlockBlobRenameScenario() throws Throwable {
-
- createEmptyFile(createTestAccount(),
- testPath);
- Path dstPath = new Path("dstFile.dat");
- fs.delete(testPath, true);
- boolean renameResult = fs.rename(testPath, dstPath);
- assertFalse(renameResult);
- }
-
- /**
- * Test basic single threaded listStatus scenario.
- */
- @Test
- public void testSingleThreadedPageBlobRenameScenario() throws Throwable {
-
- createEmptyFile(getPageBlobTestStorageAccount(),
- testPath);
- Path dstPath = new Path("dstFile.dat");
- fs.delete(testPath, true);
- boolean renameResult = fs.rename(testPath, dstPath);
- assertFalse(renameResult);
- }
-
- /**
- * Test basic single threaded listStatus scenario.
- */
- @Test
- public void testSingleThreadedBlockBlobDeleteScenario() throws Throwable {
-
- createEmptyFile(createTestAccount(),
- testPath);
- fs.delete(testPath, true);
- boolean deleteResult = fs.delete(testPath, true);
- assertFalse(deleteResult);
- }
-
- /**
- * Test basic single threaded listStatus scenario.
- */
- @Test
- public void testSingleThreadedPageBlobDeleteScenario() throws Throwable {
-
- createEmptyFile(getPageBlobTestStorageAccount(),
- testPath);
- fs.delete(testPath, true);
- boolean deleteResult = fs.delete(testPath, true);
- assertFalse(deleteResult);
- }
-
- /**
- * Test basic single threaded listStatus scenario.
- */
- @Test
- public void testSingleThreadedBlockBlobOpenScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(createTestAccount(), testPath);
- fs.delete(testPath, true);
- inputStream = fs.open(testPath);
- });
- }
-
- /**
- * Test delete then open a file.
- */
- @Test
- public void testSingleThreadedPageBlobOpenScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, ()->{
- createEmptyFile(getPageBlobTestStorageAccount(), testPath);
- fs.delete(testPath, true);
- inputStream = fs.open(testPath);
- });
- }
-
- /**
- * Attempts to write to the azure stream after it is closed will raise
- * an IOException.
- */
- @Test
- public void testWriteAfterClose() throws Throwable {
- FSDataOutputStream out = fs.create(testPath);
- out.close();
- intercept(IOException.class, STREAM_IS_CLOSED,
- () -> out.write('a'));
- intercept(IOException.class, STREAM_IS_CLOSED,
- () -> out.write(new byte[]{'a'}));
- out.hsync();
- out.flush();
- out.close();
- }
-
- @AfterEach
- public void tearDown() throws Exception {
- if (inputStream != null) {
- inputStream.close();
- }
-
- ContractTestUtils.rm(fs, testPath, true, true);
- super.tearDown();
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount()
- throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java
deleted file mode 100644
index a5ff76c2fa4b7..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.net.URI;
-import java.util.UUID;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import com.microsoft.azure.storage.CloudStorageAccount;
-import org.junit.jupiter.api.Test;
-
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
-import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig;
-
-/**
- * Test for error messages coming from SDK.
- */
-public class ITestFileSystemOperationExceptionMessage
- extends AbstractWasbTestWithTimeout {
-
-
-
- @Test
- public void testAnonymouseCredentialExceptionMessage() throws Throwable {
-
- Configuration conf = AzureBlobStorageTestAccount.createTestConfiguration();
- CloudStorageAccount account =
- AzureBlobStorageTestAccount.createTestAccount(conf);
- AzureTestUtils.assume("No test account", account != null);
-
- String testStorageAccount = verifyWasbAccountNameInConfig(conf);
- conf = new Configuration();
- conf.set("fs.AbstractFileSystem.wasb.impl",
- "org.apache.hadoop.fs.azure.Wasb");
- conf.set("fs.azure.skip.metrics", "true");
-
- String testContainer = UUID.randomUUID().toString();
- String wasbUri = String.format("wasb://%s@%s",
- testContainer, testStorageAccount);
-
- try(NativeAzureFileSystem filesystem = new NativeAzureFileSystem()) {
- filesystem.initialize(new URI(wasbUri), conf);
- fail("Expected an exception, got " + filesystem);
- } catch (Exception ex) {
-
- Throwable innerException = ex.getCause();
- while (innerException != null
- && !(innerException instanceof AzureException)) {
- innerException = innerException.getCause();
- }
-
- if (innerException != null) {
- GenericTestUtils.assertExceptionContains(String.format(
- NO_ACCESS_TO_CONTAINER_MSG, testStorageAccount, testContainer),
- ex);
- } else {
- fail("No inner azure exception");
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java
deleted file mode 100644
index 19080d031b6c9..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java
+++ /dev/null
@@ -1,381 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.FileNotFoundException;
-
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.concurrent.SubjectInheritingThread;
-
-import static org.apache.hadoop.fs.azure.ExceptionHandlingTestHelper.*;
-
-/**
- * Multithreaded operations on FS, verify failures are as expected.
- */
-public class ITestFileSystemOperationsExceptionHandlingMultiThreaded
- extends AbstractWasbTestBase {
-
- FSDataInputStream inputStream = null;
-
- private Path testPath;
- private Path testFolderPath;
-
- @BeforeEach
- @Override
- public void setUp() throws Exception {
- super.setUp();
- testPath = path("testfile.dat");
- testFolderPath = path("testfolder");
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-
- @Override
- public void tearDown() throws Exception {
-
- IOUtils.closeStream(inputStream);
- ContractTestUtils.rm(fs, testPath, true, false);
- ContractTestUtils.rm(fs, testFolderPath, true, false);
- super.tearDown();
- }
-
- /**
- * Helper method to creates an input stream to test various scenarios.
- */
- private void getInputStreamToTest(FileSystem fs, Path testPath)
- throws Throwable {
-
- FSDataOutputStream outputStream = fs.create(testPath);
- String testString = "This is a test string";
- outputStream.write(testString.getBytes());
- outputStream.close();
-
- inputStream = fs.open(testPath);
- }
-
- /**
- * Test to validate correct exception is thrown for Multithreaded read
- * scenario for block blobs.
- */
- @Test
- public void testMultiThreadedBlockBlobReadScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- NativeAzureFileSystem fs = testAccount.getFileSystem();
- Path base = methodPath();
- Path testFilePath1 = new Path(base, "test1.dat");
- Path renamePath = new Path(base, "test2.dat");
- getInputStreamToTest(fs, testFilePath1);
- Thread renameThread = new SubjectInheritingThread(
- new RenameThread(fs, testFilePath1, renamePath));
- renameThread.start();
-
- renameThread.join();
-
- byte[] readBuffer = new byte[512];
- inputStream.read(readBuffer);
- });
- }
-
- /**
- * Test to validate correct exception is thrown for Multithreaded seek
- * scenario for block blobs.
- */
- @Test
- public void testMultiThreadBlockBlobSeekScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- /*
- * AzureBlobStorageTestAccount testAccount = createTestAccount();
- * fs = testAccount.getFileSystem();
- */
- Path base = methodPath();
- Path testFilePath1 = new Path(base, "test1.dat");
- Path renamePath = new Path(base, "test2.dat");
-
- getInputStreamToTest(fs, testFilePath1);
- Thread renameThread = new SubjectInheritingThread(
- new RenameThread(fs, testFilePath1, renamePath));
- renameThread.start();
-
- renameThread.join();
-
- inputStream.seek(5);
- inputStream.read();
- });
- }
-
- /**
- * Tests basic multi threaded setPermission scenario.
- */
- @Test
- public void testMultiThreadedPageBlobSetPermissionScenario()
- throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(
- getPageBlobTestStorageAccount(),
- testPath);
- Thread t = new SubjectInheritingThread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- fs.setPermission(testPath,
- new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- }
- fs.setPermission(testPath,
- new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- });
- }
-
- /**
- * Tests basic multi threaded setPermission scenario.
- */
- @Test
- public void testMultiThreadedBlockBlobSetPermissionScenario()
- throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(createTestAccount(), testPath);
- Thread t = new SubjectInheritingThread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- fs.setPermission(testPath,
- new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- }
- fs.setPermission(testPath,
- new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- });
- }
-
- /**
- * Tests basic multi threaded setPermission scenario.
- */
- @Test
- public void testMultiThreadedPageBlobOpenScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(createTestAccount(), testPath);
- Thread t = new SubjectInheritingThread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- inputStream = fs.open(testPath);
- inputStream.close();
- }
-
- inputStream = fs.open(testPath);
- inputStream.close();
- });
- }
-
- /**
- * Tests basic multi threaded setPermission scenario.
- */
- @Test
- public void testMultiThreadedBlockBlobOpenScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(
- getPageBlobTestStorageAccount(),
- testPath);
- Thread t = new SubjectInheritingThread(new DeleteThread(fs, testPath));
- t.start();
-
- while (t.isAlive()) {
- inputStream = fs.open(testPath);
- inputStream.close();
- }
- inputStream = fs.open(testPath);
- inputStream.close();
- });
- }
-
- /**
- * Tests basic multi threaded setOwner scenario.
- */
- @Test
- public void testMultiThreadedBlockBlobSetOwnerScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(createTestAccount(), testPath);
- Thread t = new SubjectInheritingThread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- fs.setOwner(testPath, "testowner", "testgroup");
- }
- fs.setOwner(testPath, "testowner", "testgroup");
- });
- }
-
- /**
- * Tests basic multi threaded setOwner scenario.
- */
- @Test
- public void testMultiThreadedPageBlobSetOwnerScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createEmptyFile(
- getPageBlobTestStorageAccount(),
- testPath);
- Thread t = new SubjectInheritingThread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- fs.setOwner(testPath, "testowner", "testgroup");
- }
- fs.setOwner(testPath, "testowner", "testgroup");
- });
- }
-
- /**
- * Tests basic multi threaded listStatus scenario.
- */
- @Test
- public void testMultiThreadedBlockBlobListStatusScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createTestFolder(createTestAccount(), testFolderPath);
- Thread t = new SubjectInheritingThread(new DeleteThread(fs, testFolderPath));
- t.start();
- while (t.isAlive()) {
- fs.listStatus(testFolderPath);
- }
- fs.listStatus(testFolderPath);
- });
- }
-
- /**
- * Tests basic multi threaded listStatus scenario.
- */
- @Test
- public void testMultiThreadedPageBlobListStatusScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- createTestFolder(
- getPageBlobTestStorageAccount(),
- testFolderPath);
- Thread t = new SubjectInheritingThread(new DeleteThread(fs, testFolderPath));
- t.start();
- while (t.isAlive()) {
- fs.listStatus(testFolderPath);
- }
- fs.listStatus(testFolderPath);
- });
- }
-
- /**
- * Test to validate correct exception is thrown for Multithreaded read
- * scenario for page blobs.
- */
- @Test
- public void testMultiThreadedPageBlobReadScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- bindToTestAccount(getPageBlobTestStorageAccount());
- Path base = methodPath();
- Path testFilePath1 = new Path(base, "test1.dat");
- Path renamePath = new Path(base, "test2.dat");
-
- getInputStreamToTest(fs, testFilePath1);
- Thread renameThread = new SubjectInheritingThread(
- new RenameThread(fs, testFilePath1, renamePath));
- renameThread.start();
-
- renameThread.join();
- byte[] readBuffer = new byte[512];
- inputStream.read(readBuffer);
- });
- }
-
- /**
- * Test to validate correct exception is thrown for Multithreaded seek
- * scenario for page blobs.
- */
-
- @Test
- public void testMultiThreadedPageBlobSeekScenario() throws Throwable {
- assertThrows(FileNotFoundException.class, () -> {
- bindToTestAccount(getPageBlobTestStorageAccount());
-
- Path base = methodPath();
- Path testFilePath1 = new Path(base, "test1.dat");
- Path renamePath = new Path(base, "test2.dat");
-
- getInputStreamToTest(fs, testFilePath1);
- Thread renameThread = new SubjectInheritingThread(
- new RenameThread(fs, testFilePath1, renamePath));
- renameThread.start();
-
- renameThread.join();
- inputStream.seek(5);
- });
- }
-
-
- /**
- * Helper thread that just renames the test file.
- */
- private static class RenameThread implements Runnable {
-
- private final FileSystem fs;
- private final Path testPath;
- private final Path renamePath;
-
- RenameThread(FileSystem fs,
- Path testPath,
- Path renamePath) {
- this.fs = fs;
- this.testPath = testPath;
- this.renamePath = renamePath;
- }
-
- @Override
- public void run() {
- try {
- fs.rename(testPath, renamePath);
- } catch (Exception e) {
- // Swallowing the exception as the
- // correctness of the test is controlled
- // by the other thread
- }
- }
- }
-
- private static class DeleteThread implements Runnable {
- private final FileSystem fs;
- private final Path testPath;
-
- DeleteThread(FileSystem fs, Path testPath) {
- this.fs = fs;
- this.testPath = testPath;
- }
-
- @Override
- public void run() {
- try {
- fs.delete(testPath, true);
- } catch (Exception e) {
- // Swallowing the exception as the
- // correctness of the test is controlled
- // by the other thread
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationsWithThreads.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationsWithThreads.java
deleted file mode 100644
index 5a17a07e1a074..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationsWithThreads.java
+++ /dev/null
@@ -1,812 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending;
-import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Tests the Native Azure file system (WASB) using parallel threads for rename and delete operations.
- */
-public class ITestFileSystemOperationsWithThreads extends AbstractWasbTestBase {
-
- private final int renameThreads = 10;
- private final int deleteThreads = 20;
- private int iterations = 1;
- private LogCapturer logs = null;
-
- @BeforeEach
- public void setUp() throws Exception {
- super.setUp();
- Configuration conf = fs.getConf();
-
- // By default enable parallel threads for rename and delete operations.
- // Also enable flat listing of blobs for these operations.
- conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, renameThreads);
- conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, deleteThreads);
- conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, true);
-
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- // Capture logs
- logs = LogCapturer.captureLogs(LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME));
- }
-
- /*
- * Helper method to create sub directory and different types of files
- * for multiple iterations.
- */
- private void createFolder(FileSystem fs, String root) throws Exception {
- fs.mkdirs(new Path(root));
- for (int i = 0; i < this.iterations; i++) {
- fs.mkdirs(new Path(root + "/" + i));
- fs.createNewFile(new Path(root + "/" + i + "/fileToRename"));
- fs.createNewFile(new Path(root + "/" + i + "/file/to/rename"));
- fs.createNewFile(new Path(root + "/" + i + "/file+to%rename"));
- fs.createNewFile(new Path(root + "/fileToRename" + i));
- }
- }
-
- /*
- * Helper method to do rename operation and validate all files in source folder
- * doesn't exists and similar files exists in new folder.
- */
- private void validateRenameFolder(FileSystem fs, String source, String dest) throws Exception {
- // Create source folder with files.
- createFolder(fs, source);
- Path sourceFolder = new Path(source);
- Path destFolder = new Path(dest);
-
- // rename operation
- assertTrue(fs.rename(sourceFolder, destFolder));
- assertTrue(fs.exists(destFolder));
-
- for (int i = 0; i < this.iterations; i++) {
- // Check destination folder and files exists.
- assertTrue(fs.exists(new Path(dest + "/" + i)));
- assertTrue(fs.exists(new Path(dest + "/" + i + "/fileToRename")));
- assertTrue(fs.exists(new Path(dest + "/" + i + "/file/to/rename")));
- assertTrue(fs.exists(new Path(dest + "/" + i + "/file+to%rename")));
- assertTrue(fs.exists(new Path(dest + "/fileToRename" + i)));
-
- // Check source folder and files doesn't exists.
- assertFalse(fs.exists(new Path(source + "/" + i)));
- assertFalse(fs.exists(new Path(source + "/" + i + "/fileToRename")));
- assertFalse(fs.exists(new Path(source + "/" + i + "/file/to/rename")));
- assertFalse(fs.exists(new Path(source + "/" + i + "/file+to%rename")));
- assertFalse(fs.exists(new Path(source + "/fileToRename" + i)));
- }
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameSmallFolderWithThreads() throws Exception {
-
- validateRenameFolder(fs, "root", "rootnew");
-
- // With single iteration, we would have created 7 blobs.
- int expectedThreadsCreated = Math.min(7, renameThreads);
-
- // Validate from logs that threads are created.
- String content = logs.getOutput();
- assertInLog(content, "ms with threads: " + expectedThreadsCreated);
-
- // Validate thread executions
- for (int i = 0; i < expectedThreadsCreated; i++) {
- assertInLog(content,
- "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i);
- }
-
- // Also ensure that we haven't spawned extra threads.
- if (expectedThreadsCreated < renameThreads) {
- for (int i = expectedThreadsCreated; i < renameThreads; i++) {
- assertNotInLog(content,
- "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameLargeFolderWithThreads() throws Exception {
-
- // Populate source folder with large number of files and directories.
- this.iterations = 10;
- validateRenameFolder(fs, "root", "rootnew");
-
- // Validate from logs that threads are created.
- String content = logs.getOutput();
- assertInLog(content, "ms with threads: " + renameThreads);
-
- // Validate thread executions
- for (int i = 0; i < renameThreads; i++) {
- assertInLog(content,
- "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Test case for rename operation with threads disabled and flat listing enabled.
- */
- @Test
- public void testRenameLargeFolderDisableThreads() throws Exception {
- Configuration conf = fs.getConf();
-
- // Number of threads set to 0 or 1 disables threads.
- conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, 0);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- // Populate source folder with large number of files and directories.
- this.iterations = 10;
- validateRenameFolder(fs, "root", "rootnew");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Disabling threads for Rename operation as thread count 0");
-
- // Validate no thread executions
- for (int i = 0; i < renameThreads; i++) {
- String term = "AzureBlobRenameThread-"
- + Thread.currentThread().getName()
- + "-" + i;
- assertNotInLog(content, term);
- }
- }
-
- /**
- * Assert that a log contains the given term.
- * @param content log output
- * @param term search term
- */
- protected void assertInLog(String content, String term) {
- assertTrue(!content.isEmpty(), "Empty log");
- if (!content.contains(term)) {
- String message = "No " + term + " found in logs";
- LOG.error(message);
- System.err.println(content);
- fail(message);
- }
- }
-
- /**
- * Assert that a log does not contain the given term.
- * @param content log output
- * @param term search term
- */
- protected void assertNotInLog(String content, String term) {
- assertTrue(!content.isEmpty(), "Empty log");
- if (content.contains(term)) {
- String message = term + " found in logs";
- LOG.error(message);
- System.err.println(content);
- fail(message);
- }
- }
-
- /*
- * Test case for rename operation with threads and flat listing disabled.
- */
- @Test
- public void testRenameSmallFolderDisableThreadsDisableFlatListing() throws Exception {
- Configuration conf = fs.getConf();
- conf = fs.getConf();
-
- // Number of threads set to 0 or 1 disables threads.
- conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, 1);
- conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, false);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- validateRenameFolder(fs, "root", "rootnew");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Disabling threads for Rename operation as thread count 1");
-
- // Validate no thread executions
- for (int i = 0; i < renameThreads; i++) {
- assertNotInLog(content,
- "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Helper method to do delete operation and validate all files in source folder
- * doesn't exists after delete operation.
- */
- private void validateDeleteFolder(FileSystem fs, String source) throws Exception {
- // Create folder with files.
- createFolder(fs, "root");
- Path sourceFolder = new Path(source);
-
- // Delete operation
- assertTrue(fs.delete(sourceFolder, true));
- assertFalse(fs.exists(sourceFolder));
-
- for (int i = 0; i < this.iterations; i++) {
- // check that source folder and files doesn't exists
- assertFalse(fs.exists(new Path(source + "/" + i)));
- assertFalse(fs.exists(new Path(source + "/" + i + "/fileToRename")));
- assertFalse(fs.exists(new Path(source + "/" + i + "/file/to/rename")));
- assertFalse(fs.exists(new Path(source + "/" + i + "/file+to%rename")));
- assertFalse(fs.exists(new Path(source + "/fileToRename" + i)));
- }
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteSmallFolderWithThreads() throws Exception {
-
- validateDeleteFolder(fs, "root");
-
- // With single iteration, we would have created 7 blobs.
- int expectedThreadsCreated = Math.min(7, deleteThreads);
-
- // Validate from logs that threads are enabled.
- String content = logs.getOutput();
- assertInLog(content, "ms with threads: " + expectedThreadsCreated);
-
- // Validate thread executions
- for (int i = 0; i < expectedThreadsCreated; i++) {
- assertInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
-
- // Also ensure that we haven't spawned extra threads.
- if (expectedThreadsCreated < deleteThreads) {
- for (int i = expectedThreadsCreated; i < deleteThreads; i++) {
- assertNotInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteLargeFolderWithThreads() throws Exception {
- // Populate source folder with large number of files and directories.
- this.iterations = 10;
- validateDeleteFolder(fs, "root");
-
- // Validate from logs that threads are enabled.
- String content = logs.getOutput();
- assertInLog(content, "ms with threads: " + deleteThreads);
-
- // Validate thread executions
- for (int i = 0; i < deleteThreads; i++) {
- assertInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Test case for delete operation with threads disabled and flat listing enabled.
- */
- @Test
- public void testDeleteLargeFolderDisableThreads() throws Exception {
- Configuration conf = fs.getConf();
- conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, 0);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- // Populate source folder with large number of files and directories.
- this.iterations = 10;
- validateDeleteFolder(fs, "root");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Disabling threads for Delete operation as thread count 0");
-
- // Validate no thread executions
- for (int i = 0; i < deleteThreads; i++) {
- assertNotInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Test case for rename operation with threads and flat listing disabled.
- */
- @Test
- public void testDeleteSmallFolderDisableThreadsDisableFlatListing() throws Exception {
- Configuration conf = fs.getConf();
-
- // Number of threads set to 0 or 1 disables threads.
- conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, 1);
- conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, false);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- validateDeleteFolder(fs, "root");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Disabling threads for Delete operation as thread count 1");
-
- // Validate no thread executions
- for (int i = 0; i < deleteThreads; i++) {
- assertNotInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteThreadPoolExceptionFailure() throws Exception {
-
- // Spy azure file system object and raise exception for new thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
-
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenThrow(new Exception());
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- validateDeleteFolder(mockFs, "root");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content, "Failed to create thread pool with threads");
- assertInLog(content, "Serializing the Delete operation");
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteThreadPoolExecuteFailure() throws Exception {
-
- // Mock thread pool executor to throw exception for all requests.
- ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
- Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
-
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- validateDeleteFolder(mockFs, "root");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Rejected execution of thread for Delete operation on blob");
- assertInLog(content, "Serializing the Delete operation");
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteThreadPoolExecuteSingleThreadFailure() throws Exception {
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Spy a thread pool executor and link it to azure file system object.
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- // Create a thread executor and link it to mocked thread pool executor object.
- ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // Mock thread executor to throw exception for all requests.
- Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
-
- validateDeleteFolder(mockFs, "root");
-
- // Validate from logs that threads are enabled and unused threads.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Delete operation with threads 7");
- assertInLog(content,
- "6 threads not used for Delete operation on blob");
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteThreadPoolTerminationFailure() throws Exception {
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Spy a thread pool executor and link it to azure file system object.
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- ((NativeAzureFileSystem) fs).getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
-
- // Create a thread executor and link it to mocked thread pool executor object.
- // Mock thread executor to throw exception for terminating threads.
- ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
- Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
- Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException());
-
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- createFolder(mockFs, "root");
- Path sourceFolder = new Path("root");
- boolean exception = false;
- try {
- mockFs.delete(sourceFolder, true);
- } catch (IOException e){
- exception = true;
- }
-
- assertTrue(exception);
- assertTrue(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled and delete operation is failed.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Delete operation with threads");
- assertInLog(content, "Threads got interrupted Delete blob operation");
- assertInLog(content,
- "Delete failed as operation on subfolders and files failed.");
- }
-
- /*
- * Validate that when a directory is deleted recursively, the operation succeeds
- * even if a child directory delete fails because the directory does not exist.
- * This can happen if a child directory is deleted by an external agent while
- * the parent is in progress of being deleted recursively.
- */
- @Test
- public void testRecursiveDirectoryDeleteWhenChildDirectoryDeleted()
- throws Exception {
- testRecusiveDirectoryDelete(true);
- }
-
- /*
- * Validate that when a directory is deleted recursively, the operation succeeds
- * even if a file delete fails because it does not exist.
- * This can happen if a file is deleted by an external agent while
- * the parent directory is in progress of being deleted.
- */
- @Test
- public void testRecursiveDirectoryDeleteWhenDeletingChildFileReturnsFalse()
- throws Exception {
- testRecusiveDirectoryDelete(false);
- }
-
- private void testRecusiveDirectoryDelete(boolean useDir) throws Exception {
- String childPathToBeDeletedByExternalAgent = (useDir)
- ? "root/0"
- : "root/0/fileToRename";
- // Spy azure file system object and return false for deleting one file
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path(
- childPathToBeDeletedByExternalAgent)));
-
- Answer answer = new Answer() {
- public Boolean answer(InvocationOnMock invocation) throws Throwable {
- String path = (String) invocation.getArguments()[0];
- boolean isDir = (boolean) invocation.getArguments()[1];
- boolean realResult = fs.deleteFile(path, isDir);
- assertTrue(realResult);
- boolean fakeResult = false;
- return fakeResult;
- }
- };
-
- Mockito.when(mockFs.deleteFile(path, useDir)).thenAnswer(answer);
-
- createFolder(mockFs, "root");
- Path sourceFolder = new Path("root");
-
- assertTrue(mockFs.delete(sourceFolder, true));
- assertFalse(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled, that a child directory was
- // deleted by an external caller, and the parent delete operation still
- // succeeds.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Delete operation with threads");
- assertInLog(content, String.format("Attempt to delete non-existent %s %s",
- useDir ? "directory" : "file", path));
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteSingleDeleteException() throws Exception {
-
- // Spy azure file system object and raise exception for deleting one file
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0")));
- Mockito.doThrow(new IOException()).when(mockFs).deleteFile(path, true);
-
- createFolder(mockFs, "root");
- Path sourceFolder = new Path("root");
-
- boolean exception = false;
- try {
- mockFs.delete(sourceFolder, true);
- } catch (IOException e){
- exception = true;
- }
-
- assertTrue(exception);
- assertTrue(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled and delete operation failed.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Delete operation with threads");
- assertInLog(content,
- "Encountered Exception for Delete operation for file " + path);
- assertInLog(content,
- "Terminating execution of Delete operation now as some other thread already got exception or operation failed");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameThreadPoolExceptionFailure() throws Exception {
-
- // Spy azure file system object and raise exception for new thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- ((NativeAzureFileSystem) fs).getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenThrow(new Exception());
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.doReturn(mockThreadPoolExecutor).when(mockFs).getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS);
-
- validateRenameFolder(mockFs, "root", "rootnew");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content, "Failed to create thread pool with threads");
- assertInLog(content, "Serializing the Rename operation");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameThreadPoolExecuteFailure() throws Exception {
-
- // Mock thread pool executor to throw exception for all requests.
- ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
- Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- validateRenameFolder(mockFs, "root", "rootnew");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Rejected execution of thread for Rename operation on blob");
- assertInLog(content, "Serializing the Rename operation");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameThreadPoolExecuteSingleThreadFailure() throws Exception {
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Spy a thread pool executor and link it to azure file system object.
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- // Create a thread executor and link it to mocked thread pool executor object.
- ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // Mock thread executor to throw exception for all requests.
- Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
-
- validateRenameFolder(mockFs, "root", "rootnew");
-
- // Validate from logs that threads are enabled and unused threads exists.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Rename operation with threads 7");
- assertInLog(content,
- "6 threads not used for Rename operation on blob");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameThreadPoolTerminationFailure() throws Exception {
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Spy a thread pool executor and link it to azure file system object.
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- // Mock thread executor to throw exception for all requests.
- ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
- Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
- Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException());
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
-
- createFolder(mockFs, "root");
- Path sourceFolder = new Path("root");
- Path destFolder = new Path("rootnew");
- boolean exception = false;
- try {
- mockFs.rename(sourceFolder, destFolder);
- } catch (IOException e){
- exception = true;
- }
-
- assertTrue(exception);
- assertTrue(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled and rename operation is failed.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Rename operation with threads");
- assertInLog(content, "Threads got interrupted Rename blob operation");
- assertInLog(content,
- "Rename failed as operation on subfolders and files failed.");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameSingleRenameException() throws Exception {
-
- // Spy azure file system object and raise exception for deleting one file
- Path sourceFolder = new Path("root");
- Path destFolder = new Path("rootnew");
-
- // Spy azure file system object and populate rename pending spy object.
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Populate data now only such that rename pending spy object would see this data.
- createFolder(mockFs, "root");
-
- String srcKey = mockFs.pathToKey(mockFs.makeAbsolute(sourceFolder));
- String dstKey = mockFs.pathToKey(mockFs.makeAbsolute(destFolder));
-
- FolderRenamePending mockRenameFs = Mockito.spy(mockFs.prepareAtomicFolderRename(srcKey, dstKey));
- Mockito.when(mockFs.prepareAtomicFolderRename(srcKey, dstKey)).thenReturn(mockRenameFs);
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0")));
- Mockito.doThrow(new IOException()).when(mockRenameFs).renameFile(Mockito.any(FileMetadata.class));
-
- boolean exception = false;
- try {
- mockFs.rename(sourceFolder, destFolder);
- } catch (IOException e){
- exception = true;
- }
-
- assertTrue(exception);
- assertTrue(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled and delete operation failed.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Rename operation with threads");
- assertInLog(content,
- "Encountered Exception for Rename operation for file " + path);
- assertInLog(content,
- "Terminating execution of Rename operation now as some other thread already got exception or operation failed");
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestListPerformance.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestListPerformance.java
deleted file mode 100644
index 44d57d1b19f83..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestListPerformance.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-import org.junit.jupiter.api.MethodOrderer;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestMethodOrder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.azure.integration.AbstractAzureScaleTest;
-import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-
-import static org.assertj.core.api.Assumptions.assumeThat;
-
-/**
- * Test list performance.
- */
-@TestMethodOrder(MethodOrderer.Alphanumeric.class)
-public class ITestListPerformance extends AbstractAzureScaleTest {
- private static final Logger LOG = LoggerFactory.getLogger(
- ITestListPerformance.class);
-
- private static final Path TEST_DIR_PATH = new Path(
- "DirectoryWithManyFiles");
-
- private static final int NUMBER_OF_THREADS = 10;
- private static final int NUMBER_OF_FILES_PER_THREAD = 1000;
-
- private int threads;
-
- private int filesPerThread;
-
- private int expectedFileCount;
-
- @BeforeEach
- @Override
- public void setUp() throws Exception {
- super.setUp();
- Configuration conf = getConfiguration();
- // fail fast
- threads = AzureTestUtils.getTestPropertyInt(conf,
- "fs.azure.scale.test.list.performance.threads", NUMBER_OF_THREADS);
- filesPerThread = AzureTestUtils.getTestPropertyInt(conf,
- "fs.azure.scale.test.list.performance.files", NUMBER_OF_FILES_PER_THREAD);
- expectedFileCount = threads * filesPerThread;
- LOG.info("Thread = {}, Files per Thread = {}, expected files = {}",
- threads, filesPerThread, expectedFileCount);
- conf.set("fs.azure.io.retry.max.retries", "1");
- conf.set("fs.azure.delete.threads", "16");
- createTestAccount();
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create(
- "itestlistperformance",
- EnumSet.of(AzureBlobStorageTestAccount.CreateOptions.CreateContainer),
- null,
- true);
- }
-
- @Test
- public void test_0101_CreateDirectoryWithFiles() throws Exception {
- assumeThat(fs.exists(TEST_DIR_PATH)).as("Test path exists; skipping").isFalse();
-
- ExecutorService executorService = Executors.newFixedThreadPool(threads);
- CloudBlobContainer container = testAccount.getRealContainer();
-
- final String basePath = (fs.getWorkingDirectory().toUri().getPath() + "/" + TEST_DIR_PATH + "/").substring(1);
-
- ArrayList> tasks = new ArrayList<>(threads);
- fs.mkdirs(TEST_DIR_PATH);
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- for (int i = 0; i < threads; i++) {
- tasks.add(
- new Callable() {
- public Integer call() {
- int written = 0;
- for (int j = 0; j < filesPerThread; j++) {
- String blobName = basePath + UUID.randomUUID().toString();
- try {
- CloudBlockBlob blob = container.getBlockBlobReference(
- blobName);
- blob.uploadText("");
- written ++;
- } catch (Exception e) {
- LOG.error("Filed to write {}", blobName, e);
- break;
- }
- }
- LOG.info("Thread completed with {} files written", written);
- return written;
- }
- }
- );
- }
-
- List> futures = executorService.invokeAll(tasks,
- getTestTimeoutMillis(), TimeUnit.MILLISECONDS);
- long elapsedMs = timer.elapsedTimeMs();
- LOG.info("time to create files: {} millis", elapsedMs);
-
- for (Future future : futures) {
- assertTrue(future.isDone(), "Future timed out");
- assertEquals(filesPerThread, future.get().intValue(),
- "Future did not write all files timed out");
- }
- }
-
- @Test
- public void test_0200_ListStatusPerformance() throws Exception {
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- FileStatus[] fileList = fs.listStatus(TEST_DIR_PATH);
- long elapsedMs = timer.elapsedTimeMs();
- LOG.info(String.format(
- "files=%1$d, elapsedMs=%2$d",
- fileList.length,
- elapsedMs));
- Map foundInList =new HashMap<>(expectedFileCount);
-
- for (FileStatus fileStatus : fileList) {
- foundInList.put(fileStatus.getPath(), fileStatus);
- LOG.info("{}: {}", fileStatus.getPath(),
- fileStatus.isDirectory() ? "dir" : "file");
- }
- assertEquals(expectedFileCount, fileList.length,
- "Mismatch between expected files and actual");
-
-
- // now do a listFiles() recursive
- ContractTestUtils.NanoTimer initialStatusCallTimer
- = new ContractTestUtils.NanoTimer();
- RemoteIterator listing
- = fs.listFiles(TEST_DIR_PATH, true);
- long initialListTime = initialStatusCallTimer.elapsedTimeMs();
- timer = new ContractTestUtils.NanoTimer();
- while (listing.hasNext()) {
- FileStatus fileStatus = listing.next();
- Path path = fileStatus.getPath();
- FileStatus removed = foundInList.remove(path);
- assertNotNull(removed,
- "Did not find " + path + "{} in the previous listing");
- }
- elapsedMs = timer.elapsedTimeMs();
- LOG.info("time for listFiles() initial call: {} millis;"
- + " time to iterate: {} millis", initialListTime, elapsedMs);
- assertEquals(0, foundInList.size(),
- "Not all files from listStatus() were found in listFiles()");
-
- }
-
- @Test
- public void test_0300_BulkDeletePerformance() throws Exception {
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- fs.delete(TEST_DIR_PATH,true);
- long elapsedMs = timer.elapsedTimeMs();
- LOG.info("time for delete(): {} millis; {} nanoS per file",
- elapsedMs, timer.nanosPerOperation(expectedFileCount));
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
deleted file mode 100644
index 0f3d1271641e6..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.conf.Configuration;
-
-import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS;
-
-/**
- * Test class to hold all WASB authorization tests that use blob-specific keys
- * to access storage.
- */
-public class ITestNativeAzureFSAuthWithBlobSpecificKeys
- extends TestNativeAzureFileSystemAuthorization {
-
-
- @Override
- public Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- conf.set(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, "false");
- return conf;
- }
-
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
deleted file mode 100644
index 511f08f2ad179..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.jupiter.api.Test;
-
-import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACHING_ENABLE;
-
-/**
- * Test class to hold all WASB authorization caching related tests.
- */
-public class ITestNativeAzureFSAuthorizationCaching
- extends TestNativeAzureFileSystemAuthorization {
-
- private static final int DUMMY_TTL_VALUE = 5000;
-
- @Override
- public Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- conf.set(KEY_AUTH_SERVICE_CACHING_ENABLE, "true");
- return conf;
- }
-
- /**
- * Test to verify cache behavior -- assert that PUT overwrites value if present
- */
- @Test
- public void testCachePut() throws Throwable {
- CachingAuthorizer cache = new CachingAuthorizer<>(DUMMY_TTL_VALUE, "TEST");
- cache.init(createConfiguration());
- cache.put("TEST", 1);
- cache.put("TEST", 3);
- int result = cache.get("TEST");
- assertEquals(3, result, "Cache returned unexpected result");
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSPageBlobLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSPageBlobLive.java
deleted file mode 100644
index a4d8729a6804e..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSPageBlobLive.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Run the base Azure file system tests strictly on page blobs to make sure fundamental
- * operations on page blob files and folders work as expected.
- * These operations include create, delete, rename, list, and so on.
- */
-public class ITestNativeAzureFSPageBlobLive extends
- NativeAzureFileSystemBaseTest {
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount()
- throws Exception {
- Configuration conf = new Configuration();
-
- // Configure the page blob directories key so every file created is a page blob.
- conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, "/");
-
- // Configure the atomic rename directories key so every folder will have
- // atomic rename applied.
- conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES, "/");
- return AzureBlobStorageTestAccount.create(conf);
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
deleted file mode 100644
index d1ba65c5cb1fb..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-/**
- * Test append operations.
- */
-public class ITestNativeAzureFileSystemAppend extends AbstractWasbTestBase {
-
- private Path testPath;
-
- @Override
- public Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME,
- true);
- return conf;
- }
-
- @BeforeEach
- @Override
- public void setUp() throws Exception {
- super.setUp();
- testPath = methodPath();
- }
-
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create(createConfiguration());
- }
-
- /*
- * Helper method that creates test data of size provided by the
- * "size" parameter.
- */
- private static byte[] getTestData(int size) {
- byte[] testData = new byte[size];
- System.arraycopy(RandomStringUtils.randomAlphabetic(size).getBytes(), 0, testData, 0, size);
- return testData;
- }
-
- // Helper method to create file and write fileSize bytes of data on it.
- private byte[] createBaseFileWithData(int fileSize, Path testPath) throws Throwable {
-
- try(FSDataOutputStream createStream = fs.create(testPath)) {
- byte[] fileData = null;
-
- if (fileSize != 0) {
- fileData = getTestData(fileSize);
- createStream.write(fileData);
- }
- return fileData;
- }
- }
-
- /*
- * Helper method to verify a file data equal to "dataLength" parameter
- */
- private boolean verifyFileData(int dataLength, byte[] testData, int testDataIndex,
- FSDataInputStream srcStream) {
-
- try {
-
- byte[] fileBuffer = new byte[dataLength];
- byte[] testDataBuffer = new byte[dataLength];
-
- int fileBytesRead = srcStream.read(fileBuffer);
-
- if (fileBytesRead < dataLength) {
- return false;
- }
-
- System.arraycopy(testData, testDataIndex, testDataBuffer, 0, dataLength);
-
- if (!Arrays.equals(fileBuffer, testDataBuffer)) {
- return false;
- }
-
- return true;
-
- } catch (Exception ex) {
- return false;
- }
-
- }
-
- /*
- * Helper method to verify Append on a testFile.
- */
- private boolean verifyAppend(byte[] testData, Path testFile) {
-
- try(FSDataInputStream srcStream = fs.open(testFile)) {
-
- int baseBufferSize = 2048;
- int testDataSize = testData.length;
- int testDataIndex = 0;
-
- while (testDataSize > baseBufferSize) {
-
- if (!verifyFileData(baseBufferSize, testData, testDataIndex, srcStream)) {
- return false;
- }
- testDataIndex += baseBufferSize;
- testDataSize -= baseBufferSize;
- }
-
- if (!verifyFileData(testDataSize, testData, testDataIndex, srcStream)) {
- return false;
- }
-
- return true;
- } catch(Exception ex) {
- return false;
- }
- }
-
- /*
- * Test case to verify if an append on small size data works. This tests
- * append E2E
- */
- @Test
- public void testSingleAppend() throws Throwable{
-
- FSDataOutputStream appendStream = null;
- try {
- int baseDataSize = 50;
- byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, testPath);
-
- int appendDataSize = 20;
- byte[] appendDataBuffer = getTestData(appendDataSize);
- appendStream = fs.append(testPath, 10);
- appendStream.write(appendDataBuffer);
- appendStream.close();
- byte[] testData = new byte[baseDataSize + appendDataSize];
- System.arraycopy(baseDataBuffer, 0, testData, 0, baseDataSize);
- System.arraycopy(appendDataBuffer, 0, testData, baseDataSize, appendDataSize);
-
- assertTrue(verifyAppend(testData, testPath));
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- /*
- * Test case to verify append to an empty file.
- */
- @Test
- public void testSingleAppendOnEmptyFile() throws Throwable {
-
- FSDataOutputStream appendStream = null;
-
- try {
- createBaseFileWithData(0, testPath);
-
- int appendDataSize = 20;
- byte[] appendDataBuffer = getTestData(appendDataSize);
- appendStream = fs.append(testPath, 10);
- appendStream.write(appendDataBuffer);
- appendStream.close();
-
- assertTrue(verifyAppend(appendDataBuffer, testPath));
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- /*
- * Test to verify that we can open only one Append stream on a File.
- */
- @Test
- public void testSingleAppenderScenario() throws Throwable {
-
- FSDataOutputStream appendStream1 = null;
- FSDataOutputStream appendStream2 = null;
- IOException ioe = null;
- try {
- createBaseFileWithData(0, testPath);
- appendStream1 = fs.append(testPath, 10);
- boolean encounteredException = false;
- try {
- appendStream2 = fs.append(testPath, 10);
- } catch(IOException ex) {
- encounteredException = true;
- ioe = ex;
- }
-
- appendStream1.close();
-
- assertTrue(encounteredException);
- GenericTestUtils.assertExceptionContains("Unable to set Append lease on the Blob", ioe);
- } finally {
- if (appendStream1 != null) {
- appendStream1.close();
- }
-
- if (appendStream2 != null) {
- appendStream2.close();
- }
- }
- }
-
- /*
- * Tests to verify multiple appends on a Blob.
- */
- @Test
- public void testMultipleAppends() throws Throwable {
-
- int baseDataSize = 50;
- byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, testPath);
-
- int appendDataSize = 100;
- int targetAppendCount = 50;
- byte[] testData = new byte[baseDataSize + (appendDataSize*targetAppendCount)];
- int testDataIndex = 0;
- System.arraycopy(baseDataBuffer, 0, testData, testDataIndex, baseDataSize);
- testDataIndex += baseDataSize;
-
- int appendCount = 0;
-
- FSDataOutputStream appendStream = null;
-
- try {
- while (appendCount < targetAppendCount) {
-
- byte[] appendDataBuffer = getTestData(appendDataSize);
- appendStream = fs.append(testPath, 30);
- appendStream.write(appendDataBuffer);
- appendStream.close();
-
- System.arraycopy(appendDataBuffer, 0, testData, testDataIndex, appendDataSize);
- testDataIndex += appendDataSize;
- appendCount++;
- }
-
- assertTrue(verifyAppend(testData, testPath));
-
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- /*
- * Test to verify we multiple appends on the same stream.
- */
- @Test
- public void testMultipleAppendsOnSameStream() throws Throwable {
-
- int baseDataSize = 50;
- byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, testPath);
- int appendDataSize = 100;
- int targetAppendCount = 50;
- byte[] testData = new byte[baseDataSize + (appendDataSize*targetAppendCount)];
- int testDataIndex = 0;
- System.arraycopy(baseDataBuffer, 0, testData, testDataIndex, baseDataSize);
- testDataIndex += baseDataSize;
- int appendCount = 0;
-
- FSDataOutputStream appendStream = null;
-
- try {
-
- while (appendCount < targetAppendCount) {
-
- appendStream = fs.append(testPath, 50);
-
- int singleAppendChunkSize = 20;
- int appendRunSize = 0;
- while (appendRunSize < appendDataSize) {
-
- byte[] appendDataBuffer = getTestData(singleAppendChunkSize);
- appendStream.write(appendDataBuffer);
- System.arraycopy(appendDataBuffer, 0, testData,
- testDataIndex + appendRunSize, singleAppendChunkSize);
-
- appendRunSize += singleAppendChunkSize;
- }
-
- appendStream.close();
- testDataIndex += appendDataSize;
- appendCount++;
- }
-
- assertTrue(verifyAppend(testData, testPath));
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- @Test
- /*
- * Test to verify the behavior when Append Support configuration flag is set to false
- */
- public void testFalseConfigurationFlagBehavior() throws Throwable {
- assertThrows(UnsupportedOperationException.class, ()->{
- fs = testAccount.getFileSystem();
- Configuration conf = fs.getConf();
- conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME, false);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- FSDataOutputStream appendStream = null;
-
- try {
- createBaseFileWithData(0, testPath);
- appendStream = fs.append(testPath, 10);
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
-
- });
- }
-
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAtomicRenameDirList.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAtomicRenameDirList.java
deleted file mode 100644
index 75116944da450..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAtomicRenameDirList.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.hadoop.conf.Configuration;
-
-import org.junit.jupiter.api.Test;
-
-/**
- * Test atomic renaming.
- */
-public class ITestNativeAzureFileSystemAtomicRenameDirList
- extends AbstractWasbTestBase {
-
- // HBase-site config controlling HBase root dir
- private static final String HBASE_ROOT_DIR_CONF_STRING = "hbase.rootdir";
- private static final String HBASE_ROOT_DIR_VALUE_ON_DIFFERENT_FS =
- "wasb://somedifferentfilesystem.blob.core.windows.net/hbase";
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-
- @Test
- public void testAtomicRenameKeyDoesntNPEOnInitializingWithNonDefaultURI()
- throws IOException {
- NativeAzureFileSystem azureFs = fs;
- AzureNativeFileSystemStore azureStore = azureFs.getStore();
- Configuration conf = fs.getConf();
- conf.set(HBASE_ROOT_DIR_CONF_STRING, HBASE_ROOT_DIR_VALUE_ON_DIFFERENT_FS);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
- azureStore.isAtomicRenameKey("anyrandomkey");
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemClientLogging.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemClientLogging.java
deleted file mode 100644
index 7363373bc4bd0..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemClientLogging.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.net.URI;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
-
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test to validate Azure storage client side logging. Tests works only when
- * testing with Live Azure storage because Emulator does not have support for
- * client-side logging.
- *
- * Important: Do not attempt to move off commons-logging.
- * The tests will fail.
- */
-public class ITestNativeAzureFileSystemClientLogging
- extends AbstractWasbTestBase {
-
- // Core-site config controlling Azure Storage Client logging
- private static final String KEY_LOGGING_CONF_STRING = "fs.azure.storage.client.logging";
-
- // Temporary directory created using WASB.
- private static final String TEMP_DIR = "tempDir";
-
- /*
- * Helper method to verify the client logging is working. This check primarily
- * checks to make sure we see a line in the logs corresponding to the entity
- * that is created during test run.
- */
- private boolean verifyStorageClientLogs(String capturedLogs, String entity)
- throws Exception {
-
- URI uri = testAccount.getRealAccount().getBlobEndpoint();
- String container = testAccount.getRealContainer().getName();
- String validateString = uri + Path.SEPARATOR + container + Path.SEPARATOR
- + entity;
- boolean entityFound = false;
-
- StringTokenizer tokenizer = new StringTokenizer(capturedLogs, "\n");
-
- while (tokenizer.hasMoreTokens()) {
- String token = tokenizer.nextToken();
- if (token.contains(validateString)) {
- entityFound = true;
- break;
- }
- }
- return entityFound;
- }
-
- /*
- * Helper method that updates the core-site config to enable/disable logging.
- */
- private void updateFileSystemConfiguration(Boolean loggingFlag)
- throws Exception {
-
- Configuration conf = fs.getConf();
- conf.set(KEY_LOGGING_CONF_STRING, loggingFlag.toString());
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
- }
-
- // Using WASB code to communicate with Azure Storage.
- private void performWASBOperations() throws Exception {
-
- Path tempDir = new Path(Path.SEPARATOR + TEMP_DIR);
- fs.mkdirs(tempDir);
- fs.delete(tempDir, true);
- }
-
- @Test
- public void testLoggingEnabled() throws Exception {
-
- LogCapturer logs =
- LogCapturer.captureLogs(LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME));
-
- // Update configuration based on the Test.
- updateFileSystemConfiguration(true);
-
- performWASBOperations();
-
- String output = getLogOutput(logs);
- assertTrue(verifyStorageClientLogs(output, TEMP_DIR),
- "Log entry " + TEMP_DIR + " not found in " + output);
- }
-
- protected String getLogOutput(LogCapturer logs) {
- String output = logs.getOutput();
- assertTrue(!output.isEmpty(), "No log created/captured");
- return output;
- }
-
- @Test
- public void testLoggingDisabled() throws Exception {
-
- LogCapturer logs = LogCapturer.captureLogs(LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME));
-
- // Update configuration based on the Test.
- updateFileSystemConfiguration(false);
-
- performWASBOperations();
- String output = getLogOutput(logs);
-
- assertFalse(verifyStorageClientLogs(output, TEMP_DIR),
- "Log entry " + TEMP_DIR + " found in " + output);
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemConcurrencyLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemConcurrencyLive.java
deleted file mode 100644
index d8c15f4ee4829..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemConcurrencyLive.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/***
- * Test class to hold all Live Azure storage concurrency tests.
- */
-public class ITestNativeAzureFileSystemConcurrencyLive
- extends AbstractWasbTestBase {
-
- private static final int THREAD_COUNT = 102;
- private static final int TEST_EXECUTION_TIMEOUT = 30;
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-
- /**
- * Validate contract for FileSystem.create when overwrite is true and there
- * are concurrent callers of FileSystem.delete. An existing file should be
- * overwritten, even if the original destination exists but is deleted by an
- * external agent during the create operation.
- */
- @Test
- @Timeout(TEST_EXECUTION_TIMEOUT)
- public void testConcurrentCreateDeleteFile() throws Exception {
- Path testFile = methodPath();
-
- List tasks = new ArrayList<>(THREAD_COUNT);
-
- for (int i = 0; i < THREAD_COUNT; i++) {
- tasks.add(new CreateFileTask(fs, testFile));
- }
-
- ExecutorService es = null;
-
- try {
- es = Executors.newFixedThreadPool(THREAD_COUNT);
-
- List> futures = es.invokeAll(tasks);
-
- for (Future future : futures) {
- assertTrue(future.isDone());
-
- // we are using Callable, so if an exception
- // occurred during the operation, it will be thrown
- // when we call get
- assertEquals(null, future.get());
- }
- } finally {
- if (es != null) {
- es.shutdownNow();
- }
- }
- }
-
- /**
- * Validate contract for FileSystem.delete when invoked concurrently.
- * One of the threads should successfully delete the file and return true;
- * all other threads should return false.
- */
- @Test
- @Timeout(TEST_EXECUTION_TIMEOUT)
- public void testConcurrentDeleteFile() throws Exception {
- Path testFile = new Path("test.dat");
- fs.create(testFile).close();
-
- List tasks = new ArrayList<>(THREAD_COUNT);
-
- for (int i = 0; i < THREAD_COUNT; i++) {
- tasks.add(new DeleteFileTask(fs, testFile));
- }
-
- ExecutorService es = null;
- try {
- es = Executors.newFixedThreadPool(THREAD_COUNT);
-
- List> futures = es.invokeAll(tasks);
-
- int successCount = 0;
- for (Future future : futures) {
- assertTrue(future.isDone());
-
- // we are using Callable, so if an exception
- // occurred during the operation, it will be thrown
- // when we call get
- Boolean success = future.get();
- if (success) {
- successCount++;
- }
- }
-
- assertEquals(1, successCount,
- "Exactly one delete operation should return true.");
- } finally {
- if (es != null) {
- es.shutdownNow();
- }
- }
- }
-
- /**
- * Validate the bug fix for HADOOP-17089. Please note that we were never
- * able to reproduce this except during a Spark job that ran for multiple days
- * and in a hacked-up azure-storage SDK that added sleep before and after
- * the call to factory.setNamespaceAware(true) as shown in the description of
- *
- * @see https://github.com/Azure/azure-storage-java/pull/546
- */
- @Test
- @Timeout(TEST_EXECUTION_TIMEOUT)
- public void testConcurrentList() throws Exception {
- final Path testDir = new Path("/tmp/data-loss/11230174258112/_temporary/0/_temporary/attempt_20200624190514_0006_m_0");
- final Path testFile = new Path(testDir, "part-00004-15ea87b1-312c-4fdf-1820-95afb3dfc1c3-a010.snappy.parquet");
- fs.create(testFile).close();
- List tasks = new ArrayList<>(THREAD_COUNT);
-
- for (int i = 0; i < THREAD_COUNT; i++) {
- tasks.add(new ListTask(fs, testDir));
- }
-
- ExecutorService es = null;
- try {
- es = Executors.newFixedThreadPool(THREAD_COUNT);
-
- List> futures = es.invokeAll(tasks);
-
- for (Future future : futures) {
- assertTrue(future.isDone());
-
- // we are using Callable, so if an exception
- // occurred during the operation, it will be thrown
- // when we call get
- long fileCount = future.get();
- assertEquals(1, fileCount, "The list should always contain 1 file.");
- }
- } finally {
- if (es != null) {
- es.shutdownNow();
- }
- }
- }
-
- abstract class FileSystemTask implements Callable {
- private final FileSystem fileSystem;
- private final Path path;
-
- FileSystem getFileSystem() {
- return this.fileSystem;
- }
-
- Path getFilePath() {
- return this.path;
- }
-
- FileSystemTask(FileSystem fs, Path p) {
- this.fileSystem = fs;
- this.path = p;
- }
-
- public abstract V call() throws Exception;
- }
-
- class DeleteFileTask extends FileSystemTask {
-
- DeleteFileTask(FileSystem fs, Path p) {
- super(fs, p);
- }
-
- @Override
- public Boolean call() throws Exception {
- return this.getFileSystem().delete(this.getFilePath(), false);
- }
- }
-
- class CreateFileTask extends FileSystemTask