Capture Azure block blob traffic with fiddler - azure

Question

I have an issue with block blob storage of Azure. The issue is that If I try to access the storage and create a container from a jar, it works fine. If I try to run it from a spark-submit command, it doesn't work. I'am trying to capture the traffic between my code and Azure to see where it goes wrong but the problem is that fiddler doesn't capture that kind of traffic although I can capture traffic when accessing other sites like www.google.com.
this works:
import java.net.*;
import java.io.*;
public class Example
{
public static void main(String[] args) throws Exception
{
System.setProperty("proxySet", "true");
System.setProperty("proxyHost", "127.0.0.1");
System.setProperty("proxyPort", "9090");
System.setProperty("javax.net.ssl.trustStore", "C:\\data\\keys\\FiddlerKeystore");
System.setProperty("javax.net.ssl.trustStorePassword", "password");
URL x = new URL("https://www.google.com");
HttpURLConnection hc = (HttpURLConnection)x.openConnection();
hc.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2");
InputStream is = hc.getInputStream();
int u = 0;
byte[] kj = new byte[1024];
while((u = is.read(kj)) != -1)
{
System.out.write(kj,0,u);
}
is.close();
}
}
Now, If I do the same with Azure code Fiddler doesn't capture anything:
Here is my Azure code:
import azure.AzureBlockBlobClient;
import common.AzureConf;
import org.apache.log4j.BasicConfigurator;
import java.io.IOException;
public class AzureExample {
private AzureBlockBlobClient azureBlockBlobClient;
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(AzureExample.class);
public AzureExample() {
azureBlockBlobClient = new AzureBlockBlobClient(AzureConf.ACCOUNT_NAME,AzureConf.ACCOUNT_KEY, AzureConf.CONTAINER_NAME);
azureBlockBlobClient.createContainer();
}
public static void main(String... args) throws IOException {
BasicConfigurator.configure();
System.setProperty("proxySet", "true");
System.setProperty("proxyHost", "127.0.0.1");
System.setProperty("proxyPort", "9090");
System.setProperty("javax.net.ssl.trustStore", "C:\\data\\keys\\FiddlerKeystore");
System.setProperty("javax.net.ssl.trustStorePassword", "password");
new AzureExample();
System.exit(0);
}
}
Here is the client the connects to Azure:
public AzureBlockBlobClient(String accountName, String accountKey, String containerName) {
this.accountName = accountName;
this.accountKey = accountKey;
this.containerName = containerName;
init();
}
private void init() {
log.info("Init AzureBlockBlobClient started...");
try {
SharedKeyCredentials creds = new SharedKeyCredentials(accountName, accountKey);
serviceURL = new ServiceURL(new URL("https://" + accountName + ".blob.core.windows.net/"),
StorageURL.createPipeline(creds, new PipelineOptions()));
containerURL = serviceURL.createContainerURL(containerName);
}catch (InvalidKeyException e){
log.error("Authentication error while trying to access storage account", e);
}catch (MalformedURLException e) {
log.error("Invalid Service URL", e);
e.printStackTrace();
}catch (Exception e) {
e.printStackTrace();
log.error("Error initializing AzureBlockBlobClient", e);
}
log.info("Init AzureBlockBlobClient Done!");
}
public void createContainer(){
try {
// Let's create a container using a blocking call to Azure Storage
// If container exists, we'll catch and continue
log.info("Creating container {}." , containerName);
ContainerCreateResponse response = containerURL.create(null, null, null).blockingGet();
log.info("Container Create Response was {}." , response.statusCode());
}
catch (RestException e){
if (e instanceof RestException && e.response().statusCode() != 409) {
log.error("Error Creating container", e);
} else {
log.info("Container {} already exists, resuming...", containerName);
}
}
}
And this is where my constants are:
public interface AzureConf {
String ACCOUNT_KEY ="<SomeAccountKey>";
String ACCOUNT_NAME = "storage";
String CONTAINER_NAME = "My-container";
}
This is my maven pom.xml file:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>examples</groupId>
<artifactId>spark-azure-storage</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<junit.version>4.12</junit.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
<version>2.7.1</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
<version>2.0.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<version>10.1.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>io.reactivex.rxjava2</groupId>
<artifactId>rxjava</artifactId>
<version>2.2.3</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.16</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.typesafe.akka/akka-actor -->
<dependency>
<groupId>com.microsoft.rest.v2</groupId>
<artifactId>client-runtime</artifactId>
<version>2.0.0</version>
<!--I have to exclude following dependencies and include version 2.9.7 of them otherwise I get
SoSuchMethodError-->
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.16</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.9.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.9.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.7</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.2.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"></transformer>
</transformers>
</configuration>
</plugin>
</plugins>
</build>
Any help to get this to work?
Thank you in advance

Answer 1

According to the Oracle offical document Java Networking and Proxies, there is not these properties proxySet, proxyHost and proxyPort for Java System Properties.
Please use https.proxyHost and https.proxyPort instead of them, which work for me.