After adding some new secrets to Terraform using the 1Password provider, we saw an error without much helpful output.
$ terraform plan
...
Error: rpc error: code = Unavailable desc = transport is closing
Error: rpc error: code = Canceled desc = context canceled
...
Terraform provider:
terraform {
required_providers {
onepassword = {
source = "anasinnyk/onepassword"
version = "~> 1.2.1"
}
}
required_version = "~> 0.13"
}
Terraform yml:
data "onepassword_item_password" "search_cloud_id" {
name = "Azure Elastic Cloud ID"
vault = data.onepassword_vault.vault_name.id
}
data "onepassword_item_password" "search_api_key" {
name = "Azure Elastic Cloud API key"
vault = data.onepassword_vault.vault_name.id
}
resource "kubernetes_secret" "search" {
metadata {
name = "search"
namespace = kubernetes_namespace.production.id
}
data = {
"ELASTICSEARCH_CLOUD_ID" = data.onepassword_item_password.api_search_cloud_id.password
"ELASTICSEARCH_API_KEY" = data.onepassword_item_password.api_search_api_key.password
}
type = "Opaque"
}
We managed to get some useful output by removing one data reference at a time, which lead to the errors printing:
panic: runtime error: invalid memory address or nil pointer dereference
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: [signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0x147d1bd]
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1:
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: goroutine 194 [running]:
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/anasinnyk/terraform-provider-1password/onepassword.resourceItemPasswordRead(0x19418a0, 0xc0004ac540, 0xc000096f80, 0x173d040, 0xc0007ac740, 0xc0003bce40, 0xc000119910, 0x100c9b8)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/anasinnyk/terraform-provider-1password/onepassword/resource_item_password.go:75 +0x18d
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*Resource).read(0xc0004613f0, 0x1941820, 0xc000384300, 0xc000096f80, 0x173d040, 0xc0007ac740, 0x0, 0x0, 0x0)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2#v2.0.0/helper/schema/resource.go:288 +0x1ec
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema.(*Resource).ReadDataApply(0xc0004613f0, 0x1941820, 0xc000384300, 0xc000304b80, 0x173d040, 0xc0007ac740, 0xc0007ac740, 0xc000304b80, 0x0, 0x0)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2#v2.0.0/helper/schema/resource.go:489 +0xff
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2/internal/helper/plugin.(*GRPCProviderServer).ReadDataSource(0xc00026e6a0, 0x1941820, 0xc000384300, 0xc0003842c0, 0xc00026e6a0, 0xc00026e6b0, 0x185a058)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2#v2.0.0/internal/helper/plugin/grpc_provider.go:1102 +0x4c5
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2/internal/tfplugin5._Provider_ReadDataSource_Handler.func1(0x1941820, 0xc000384300, 0x17dcd60, 0xc0003842c0, 0xc000384300, 0x1773c80, 0xc0004ac401, 0xc000304640)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2#v2.0.0/internal/tfplugin5/tfplugin5.pb.go:3348 +0x86
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2/plugin.Serve.func3.1(0x19418e0, 0xc0003d4480, 0x17dcd60, 0xc0003842c0, 0xc000304620, 0xc000304640, 0xc0007c8ba0, 0x11b81c8, 0x17c7a20, 0xc0003d4480)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2#v2.0.0/plugin/serve.go:76 +0x87
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2/internal/tfplugin5._Provider_ReadDataSource_Handler(0x17fdb60, 0xc00026e6a0, 0x19418e0, 0xc0003d4480, 0xc0004ac4e0, 0xc00000d080, 0x19418e0, 0xc0003d4480, 0xc000010090, 0x90)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: github.com/hashicorp/terraform-plugin-sdk/v2#v2.0.0/internal/tfplugin5/tfplugin5.pb.go:3350 +0x14b
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: google.golang.org/grpc.(*Server).processUnaryRPC(0xc00027ae00, 0x1949c60, 0xc000103380, 0xc00018e000, 0xc00020acf0, 0x1e49910, 0x0, 0x0, 0x0)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: google.golang.org/grpc#v1.30.0/server.go:1171 +0x50a
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: google.golang.org/grpc.(*Server).handleStream(0xc00027ae00, 0x1949c60, 0xc000103380, 0xc00018e000, 0x0)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: google.golang.org/grpc#v1.30.0/server.go:1494 +0xccd
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0000382e0, 0xc00027ae00, 0x1949c60, 0xc000103380, 0xc00018e000)
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: google.golang.org/grpc#v1.30.0/server.go:834 +0xa1
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: created by google.golang.org/grpc.(*Server).serveStreams.func1
2021-08-27T15:34:29.367+0930 [DEBUG] plugin.terraform-provider-onepassword_v1.2.1: google.golang.org/grpc#v1.30.0/server.go:832 +0x204
2021-08-27T15:34:29.368+0930 [WARN] plugin.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing"
2021/08/27 15:34:29 [ERROR] eval: *terraform.evalReadDataRefresh, err: rpc error: code = Unavailable desc = transport is closing
2021/08/27 15:34:29 [ERROR] eval: *terraform.evalReadDataRefresh, err: rpc error: code = Unavailable desc = transport is closing
2021/08/27 15:34:29 [ERROR] eval: *terraform.evalReadDataRefresh, err: rpc error: code = Unavailable desc = transport is closing
2021/08/27 15:34:29 [ERROR] eval: *terraform.EvalSequence, err: rpc error: code = Unavailable desc = transport is closing
2021-08-27T15:34:29.369+0930 [DEBUG] plugin: plugin process exited: path=.terraform/plugins/registry.terraform.io/anasinnyk/onepassword/1.2.1/darwin_amd64/terraform-provider-onepassword_v1.2.1 pid=17549 error="exit status 2"
2021/08/27 15:34:29 [ERROR] eval: *terraform.EvalSequence, err: rpc error: code = Unavailable desc = transport is closing
2021/08/27 15:34:29 [TRACE] [walkRefresh] Exiting eval tree: data.onepassword_item_password.search_api_key
2021/08/27 15:34:29 [ERROR] eval: *terraform.EvalSequence, err: rpc error: code = Unavailable desc = transport is closing
2021/08/27 15:34:29 [TRACE] vertex "data.onepassword_item_password.search_api_key": visit complete
2021/08/27 15:34:29 [TRACE] vertex "data.onepassword_item_password.search_api_key": dynamic subgraph encountered errors
2021/08/27 15:34:29 [TRACE] vertex "data.onepassword_item_password.search_api_key": visit complete
2021/08/27 15:34:29 [TRACE] vertex "data.onepassword_item_password.search_api_key (expand)": dynamic subgraph encountered errors
2021/08/27 15:34:29 [TRACE] vertex "data.onepassword_item_password.search_api_key (expand)": visit complete
2021/08/27 15:34:29 [TRACE] dag/walk: upstream of "provider[\"registry.terraform.io/hashicorp/kubernetes\"] (close)" errored, so skipping
2021/08/27 15:34:29 [TRACE] dag/walk: upstream of "provider[\"registry.terraform.io/anasinnyk/onepassword\"] (close)" errored, so skipping
2021/08/27 15:34:29 [TRACE] dag/walk: upstream of "root" errored, so skipping
2021-08-27T15:34:29.501+0930 [DEBUG] plugin: plugin exited
2021-08-27T15:34:29.502+0930 [WARN] plugin.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing"
2021-08-27T15:34:29.507+0930 [DEBUG] plugin: plugin process exited: path=.terraform/plugins/registry.terraform.io/hashicorp/kubernetes/1.13.3/darwin_amd64/terraform-provider-kubernetes_v1.13.3_x4 pid=17673
2021-08-27T15:34:29.507+0930 [DEBUG] plugin: plugin exited
!!!!!!!!!!!!!!!!!!!!!!!!!!! TERRAFORM CRASH !!!!!!!!!!!!!!!!!!!!!!!!!!!!
Terraform crashed! This is always indicative of a bug within Terraform.
A crash log has been placed at "crash.log" relative to your current
working directory. It would be immensely helpful if you could please
report the crash with Terraform[1] so that we can fix this.
When reporting bugs, please include your terraform version. That
information is available on the first line of crash.log. You can also
get it by running 'terraform --version' on the command line.
SECURITY WARNING: the "crash.log" file that was created may contain
sensitive information that must be redacted before it is safe to share
on the issue tracker.
[1]: https://github.com/hashicorp/terraform/issues
!!!!!!!!!!!!!!!!!!!!!!!!!!! TERRAFORM CRASH !!!!!!!!!!!!!!!!!!!!!!!!!!!!
This led us to find that one of our team members managed to create two 1Password entries with the same name in the same vault.
After deleting the duplicate entry in 1Password, terraform plan ran without error again.
Related
What specific syntax or configuration changes must be made in order to resolve the error below in which terraform is failing to create an instance of azuread_application?
THE CODE:
The terraform code that is triggering the error when terraform apply is run is as follows:
variable "tenantId" { }
variable "clientId" { }
variable "clientSecret" { }
variable "instanceName" { }
terraform {
required_providers {
azuread = {
source = "hashicorp/azuread"
version = "2.5.0"
}
}
}
provider "azuread" {
tenant_id = var.tenantId
client_id = var.clientId
client_secret = var.clientSecret
}
resource "azuread_application" "appRegistration" {
display_name = var.instanceName
app_role {
allowed_member_types = ["User", "Application"]
description = "Admins can manage roles and perform all task actions"
display_name = "Admin"
enabled = true
id = "1b19509b-32b1-4e9f-b71d-4992aa991967"
value = "admin"
}
}
THE ERROR:
The error and log output that result from running the above code with terraform apply are:
2021/10/05 17:47:18 [DEBUG] module.ad-admin.azuread_application.appRegistration:
apply errored, but we're indicating that via the Error pointer rather than returning it:
Could not create application: json.Marshal():
json: error calling MarshalJSON for type msgraph.Application:
json: error calling MarshalJSON for type *msgraph.Owners: marshaling Owners: encountered DirectoryObject with nil ODataId
2021/10/05 17:47:18 [TRACE] EvalMaybeTainted: module.ad-admin.azuread_application.appRegistration encountered an error during creation, so it is now marked as tainted
2021/10/05 17:47:18 [TRACE] EvalWriteState: removing state object for module.ad-admin.azuread_application.appRegistration
2021/10/05 17:47:18 [TRACE] EvalApplyProvisioners: azuread_application.appRegistration has no state, so skipping provisioners
2021/10/05 17:47:18 [TRACE] EvalMaybeTainted: module.ad-admin.azuread_application.appRegistration encountered an error during creation, so it is now marked as tainted
2021/10/05 17:47:18 [TRACE] EvalWriteState: removing state object for module.ad-admin.azuread_application.appRegistration
2021/10/05 17:47:18 [TRACE] vertex "module.ad-admin.azuread_application.appRegistration": visit complete
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin.output.application_id (expand)" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin.azuread_service_principal.appRegistrationSP" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "output.application_id" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin.output.appId (expand)" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin.azuread_service_principal_password.appRegistrationSP_pwd" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "output.appId" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin.azurerm_role_assignment.appRegistrationSP_role_assignment_vault" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin.azurerm_role_assignment.appRegistrationSP_role_assignment" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin.provider[\"registry.terraform.io/hashicorp/azuread\"] (close)" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin.provider[\"registry.terraform.io/hashicorp/azurerm\"] (close)" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "module.ad-admin (close)" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "meta.count-boundary (EachMode fixup)" errored, so skipping
2021/10/05 17:47:18 [TRACE] dag/walk: upstream of "root" errored, so skipping
2021/10/05 17:47:18 [TRACE] statemgr.Filesystem: creating backup snapshot at terraform.tfstate.backup
2021/10/05 17:47:18 [TRACE] statemgr.Filesystem: state has changed since last snapshot, so incrementing serial to 391
2021/10/05 17:47:18 [TRACE] statemgr.Filesystem: writing snapshot at terraform.tfstate
2021/10/05 17:47:18 [TRACE] statemgr.Filesystem: removing lock metadata file .terraform.tfstate.lock.info
Error: Could not create application
on ..\..\..\..\modules\ad-admin\active-directory.tf line 69, in resource "azuread_application" "appRegistration":
69: resource "azuread_application" "appRegistration" {
json.Marshal(): json: error calling MarshalJSON for type msgraph.Application:
json: error calling MarshalJSON for type *msgraph.Owners: marshaling Owners:
2021/10/05 17:47:18 [TRACE] statemgr.Filesystem: unlocked by closing terraform.tfstate
encountered DirectoryObject with nil ODataId
terraform -version gives:
Terraform v1.0.8
on windows_amd64
This was a bug, reported as GitHub issue:
Error: ODataId was nil when creating an azuread_group resource #588
The resolution to the problem in the OP is to upgrade the version from 2.5.0 to 2.6.0 in the required_providers block from the code in the OP above as follows:
terraform {
required_providers {
azuread = {
source = "hashicorp/azuread"
version = "2.6.0"
}
}
}
The fix was released, and it is working for me.
Workaround for corrupted or missing #odata.id for directory objects
I'm developing a nginx (1.19.0) body filter module with multi-threading enabled (--with-threads Enables NGINX to use thread pools. For details, see Thread Pools in NGINX Boost Performance 9x! on the NGINX blog.), which aims to save the acess_token in the response from the upstream server.
I referred to the development guide - Threads, How to make Nginx wait for a thread pool task and nginx HTTP module with Thread Pools and Tasks,
My code snippet is as following:
typedef struct {
int status;
cJSON *oauth2_rsp;
ngx_http_request_t *req;
ngx_chain_t *chain;
} redis_thread_ctx_t;
/* This function is executed in a separate thread */
static void redis_thread_func(void *data, ngx_log_t *log) {
ngx_logd("SAM_DEBUG: redis_thread_func");
redis_thread_ctx_t *ctx = data;
cJSON *oauth2_access_token = cJSON_GetObjectItemCaseSensitive(ctx->oauth2_rsp, OAUTH2_PARAM_NAME_ACCESS_TOKEN);
cJSON *oauth2_token_type = cJSON_GetObjectItemCaseSensitive(ctx->oauth2_rsp, OAUTH2_PARAM_NAME_TOKEN_TYPE);
cJSON *oauth2_expires_in = cJSON_GetObjectItemCaseSensitive(ctx->oauth2_rsp, OAUTH2_PARAM_NAME_EXPIRES_IN);
if (0 == cache_token(ctx->req, oauth2_access_token->valuestring,
cJSON_IsString(oauth2_token_type) ? oauth2_token_type->valuestring : "Bear",
cJSON_IsNumber(oauth2_expires_in) ? oauth2_expires_in->valueint : 3600)) {
ctx->status = NGX_HTTP_OK;
} else {
ngx_log_error(NGX_LOG_ERR, log, 0, "cache_token failed");
}
ngx_logd("SAM_DEBUG: after cache_token");
cJSON_free(ctx->oauth2_rsp);
ngx_logd("SAM_DEBUG: after cJSON_free");
}
/*
* The task completion handler executes on the main event loop, and is pretty straightforward: Mark the background
* processing complete, and call the nginx HTTP function to resume processing of the request.
*/
static void redis_thread_completion(ngx_event_t *ev) {
redis_thread_ctx_t *ctx = ev->data;
ngx_http_request_t *req = ctx->req;
ngx_connection_t *con = req->connection;
ngx_log_t *log = con->log;
ngx_http_set_log_request(log, req);
ngx_logd("SAM_DEBUG: redis_thread_completion: \"%V?%V\"", &req->uri, &req->args);
req->main->blocked--;
req->aio = 0;
//ngx_http_handler(req);
ngx_http_next_body_filter(req, ctx->chain);
//ngx_http_finalize_request(req, NGX_DONE);
ngx_logd("SAM_DEBUG: after ngx_http_next_body_filter");
}
//https://serverfault.com/questions/480352/modify-data-being-proxied-by-nginx-on-the-fly
static ngx_int_t ngx_http_pep_body_filter(ngx_http_request_t *req, ngx_chain_t *chain) {
// ... omitted for brevity
cJSON *oauth2_rsp_json = NULL;
//#if (NGX_THREADS)
ngx_thread_task_t *task = ngx_thread_task_alloc(req->pool, sizeof(redis_thread_ctx_t));
if (NULL == task) {
return NGX_ERROR;
}
ngx_logd("SAM_DEBUG: after ngx_thread_task_alloc");
redis_thread_ctx_t *redis_ctx = task->ctx;
redis_ctx->status = NGX_HTTP_BAD_GATEWAY;
redis_ctx->req = req;
redis_ctx->oauth2_rsp = oauth2_rsp_json;
redis_ctx->chain = chain;
task->handler = redis_thread_func;
task->event.handler = redis_thread_completion;
task->event.data = redis_ctx;
ngx_http_core_loc_conf_t *clcf = ngx_http_get_module_loc_conf(req, ngx_http_core_module);
//subrequests=51, count=1, blocked=1, aio=0
ngx_logd("SAM_DEBUG: subrequests=%d, count=%d, blocked=%d, aio=%d", req->subrequests, req->count, req->blocked, req->aio);
if (NGX_OK != ngx_thread_task_post(clcf->thread_pool, task)) {
req->main->blocked--;
cJSON_free(oauth2_rsp_json);
ngx_log_error(NGX_LOG_ERR, log, 0, "ngx_thread_task_post failed");
return NGX_ERROR; //NGX_HTTP_INTERNAL_SERVER_ERROR
}
//Note: increment `req->main->blocked` so nginx won't finalize request (req)
req->main->blocked++;
req->aio = 1;
ngx_logd("SAM_DEBUG: after ngx_thread_task_post");
//#else
#if defined(USE_REDIS_TO_CACHE_TOKEN) && (NGX_THREADS)
return NGX_OK; //NGX_AGAIN
#else
return ngx_http_next_body_filter ? ngx_http_next_body_filter(req, chain) : NGX_OK;
#endif
}
After a test, unfortunately, I found the client could not receive the reponse. Postman showed me "Error: socket hang up" or there's no corresponding HTTP response paket in wireshak. In addition, error.log was as following,
2020/07/20 18:38:55 [debug] 461#461: *3 redis_thread_completion|772|SAM_DEBUG: after ngx_http_next_body_filter
2020/07/20 18:38:55 [debug] 461#461: timer delta: 3
2020/07/20 18:38:55 [debug] 461#461: worker cycle
2020/07/20 18:38:55 [debug] 461#461: epoll timer: 59997
2020/07/20 18:39:55 [debug] 461#461: timer delta: 59998
2020/07/20 18:39:55 [debug] 461#461: *3 event timer del: 3: 18416868
2020/07/20 18:39:55 [debug] 461#461: *3 http empty handler
2020/07/20 18:39:55 [debug] 461#461: worker cycle
2020/07/20 18:39:55 [debug] 461#461: epoll timer: 5002
2020/07/20 18:40:00 [debug] 461#461: timer delta: 5003
2020/07/20 18:40:00 [debug] 461#461: *3 event timer del: 3: 18421871
2020/07/20 18:40:00 [debug] 461#461: *3 http keepalive handler
2020/07/20 18:40:00 [debug] 461#461: *3 close http connection: 3
2020/07/20 18:40:00 [debug] 461#461: *3 reusable connection: 0
2020/07/20 18:40:00 [debug] 461#461: *3 free: 0000000000000000
2020/07/20 18:40:00 [debug] 461#461: *3 free: 00007FFFEC420BF0, unused: 136
2020/07/20 18:40:00 [debug] 461#461: worker cycle
2020/07/20 18:40:00 [debug] 461#461: epoll timer: -1
Where I have gone wrong?
After reading the code of static void ngx_http_upstream_thread_event_handler(ngx_event_t *ev) in nginx\src\http\ngx_http_upstream.c, I changed ngx_http_next_body_filter(req, ctx->chain); to req->write_event_handler(req); in static void redis_thread_completion(ngx_event_t *ev) and
static ngx_int_t ngx_http_pep_body_filter(ngx_http_request_t *req, ngx_chain_t *chain) {
// ... omitted for brevity
return ngx_http_next_body_filter(req, chain);
}
As a result, nginx could send the HTTP response to the client.
I am trying out Attribute based access control using GoLang and facing an issue in getting the attribute in the chaincode. I'm using cid.GetAttributeValue() function and the function returns nothing. It prints blank and the found variable is set to false.
The things I have done are:
Registered a user defined attribute named "firstName" with value "*" to the admin. (Also tried setting it to a specific value)
Enrolled that admin to the network
Registered and enrolled the user with the same attribute "firstName" (with ecert flag set to true)
Invoked a chaincode to get the attribute value using the enrolled user
Snippets of relevant code have been attached below:
Fabric-CA-Server Config File
identities:
- name: admin
pass: adminpw
type: client
affiliation: ""
attrs:
hf.Registrar.Roles: "*"
hf.Registrar.DelegateRoles: "*"
hf.Revoker: true
hf.IntermediateCA: true
hf.GenCRL: true
hf.Registrar.Attributes: "firstName"
hf.AffiliationMgr: true
firstName: "*"
Code for registering and enrolling an user.
user = {
enrollmentID: username,
affiliation: orgName + '.department1',
maxEnrollments: 5,
role: 'client',
attrs: [{ name: 'firstName', value: 'Mathan_sir', ecert: true }]
}
console.log(user);
var enrollmentSecret = await fabric_CA_Client.register(user, admin);
console.log('client registered');
console.log('Successfully registered user - secret:' + enrollmentSecret);
var enrollment = await fabric_CA_Client.enroll({
enrollmentID: username,
enrollmentSecret: enrollmentSecret,
attr_reqs: [
{ name: "firstName", optional: false }
// { name: "lastName", optional: false }
]
});
Chaincode.
value, found, err := cid.GetAttributeValue(stub, "firstName")
if err != nil {
return shim.Error("could not do abac")
}
fmt.Println("value is ", value)
fmt.Println("found is ", found)
Log in Fabric - CA :
2018/07/17 04:23:00 [INFO] Created default configuration file at /etc/hyperledger/fabric-ca-server/fabric-ca-server-config.yaml
2018/07/17 04:23:00 [INFO] Starting server in home directory: /etc/hyperledger/fabric-ca-server
2018/07/17 04:23:00 [INFO] Server Version: 1.2.0-stable
2018/07/17 04:23:00 [INFO] Server Levels: &{Identity:1 Affiliation:1 Certificate:1 Credential:1 RAInfo:1 Nonce:1}
2018/07/17 04:23:00 [DEBUG] Default CA configuration, if necessary, will be used to replace missing values for additional CAs: {Version:1.2.0-stable Cfg:{Identities:{AllowRemove:false} Affiliations:{AllowRemove:false}} CA:{Name: Keyfile: Certfile:ca-cert.pem Chainfile:ca-chain.pem} Signing:0xc420555bc0 CSR:{CN:fabric-ca-server Names:[{C:US ST:North Carolina L: O:Hyperledger OU:Fabric SerialNumber:}] Hosts:[e3c7da43b062 localhost] KeyRequest:<nil> CA:0xc4200ffec0 SerialNumber:} Registry:{MaxEnrollments:-1 Identities:[{ Name:**** Pass:**** Type:client Affiliation: MaxEnrollments:0 Attrs:map[hf.GenCRL:1 hf.Registrar.Attributes:* hf.AffiliationMgr:1 hf.Registrar.Roles:* hf.Registrar.DelegateRoles:* hf.Revoker:1 hf.IntermediateCA:1] }]} Affiliations:map[org2:[department1] org1:[department1 department2]] LDAP:{ Enabled:false URL:ldap://****:****#<host>:<port>/<base> UserFilter:(uid=%s) GroupFilter:(memberUid=%s) Attribute:{[uid member] [{ }] map[groups:[{ }]]} TLS:{false [] { }} } DB:{ Type:sqlite3 Datasource:fabric-ca-server.db TLS:{false [] { }} } CSP:0xc4204e5080 Client:<nil> Intermediate:{ParentServer:{ URL: CAName: } TLS:{Enabled:false CertFiles:[] Client:{KeyFile: CertFile:}} Enrollment:{ Name: Secret:**** CAName: AttrReqs:[] Profile: Label: CSR:<nil> Type:x509 }} CRL:{Expiry:24h0m0s} Idemix:{IssuerPublicKeyfile: IssuerSecretKeyfile: RevocationPublicKeyfile: RevocationPrivateKeyfile: RHPoolSize:0 NonceExpiration: NonceSweepInterval:}}
2018/07/17 04:23:00 [DEBUG] Additional CAs to be started: [/etc/hyperledger/fabric-ca-server/ca/ca-config.yaml]
2018/07/17 04:23:00 [INFO] Loading CA from /etc/hyperledger/fabric-ca-server/ca/ca-config.yaml
2018/07/17 04:23:00 [DEBUG] CA configuration after checking for missing values: &{Version:1.1.0 Cfg:{Identities:{AllowRemove:false} Affiliations:{AllowRemove:false}} CA:{Name:ca-org1 Keyfile:/etc/hyperledger/fabric-ca-server-config/3099498edb13f731965246b01e74514e4b80e978a058cf4cb9b99ab1408e2569_sk Certfile:/etc/hyperledger/fabric-ca-server-config/ca.org1.example.com-cert.pem Chainfile:ca-chain.pem} Signing:0xc4205a0e50 CSR:{CN:fabric-ca-server Names:[{C:US ST:North Carolina L: O:Hyperledger OU:Fabric SerialNumber:}] Hosts:[e3c7da43b062 localhost] KeyRequest:<nil> CA:0xc42059cea0 SerialNumber:} Registry:{MaxEnrollments:-1 Identities:[{ Name:**** Pass:**** Type:client Affiliation: MaxEnrollments:0 Attrs:map[hf.Registrar.Roles:* hf.Registrar.DelegateRoles:* hf.Revoker:1 hf.IntermediateCA:1 hf.GenCRL:1 hf.Registrar.Attributes:firstName hf.AffiliationMgr:1 firstName:*] }]} Affiliations:map[org2:[department1] org1:[department1 department2]] LDAP:{ Enabled:false URL:ldap://****:****#<host>:<port>/<base> UserFilter:(uid=%s) GroupFilter:(memberUid=%s) Attribute:{[uid member] [{ }] map[groups:[{ }]]} TLS:{false [] { }} } DB:{ Type:sqlite3 Datasource:fabric-ca-server.db TLS:{false [] { }} } CSP:0xc42059eba0 Client:<nil> Intermediate:{ParentServer:{ URL: CAName: } TLS:{Enabled:false CertFiles:[] Client:{KeyFile: CertFile:}} Enrollment:{ Name: Secret:**** CAName: AttrReqs:[] Profile: Label: CSR:<nil> Type:x509 }} CRL:{Expiry:24h0m0s} Idemix:{IssuerPublicKeyfile: IssuerSecretKeyfile: RevocationPublicKeyfile: RevocationPrivateKeyfile: RHPoolSize:1000 NonceExpiration:15s NonceSweepInterval:15m}}
2018/07/17 04:23:00 [DEBUG] Init CA with home /etc/hyperledger/fabric-ca-server/ca and config {Version:1.1.0 Cfg:{Identities:{AllowRemove:false} Affiliations:{AllowRemove:false}} CA:{Name:ca-org1 Keyfile:/etc/hyperledger/fabric-ca-server-config/3099498edb13f731965246b01e74514e4b80e978a058cf4cb9b99ab1408e2569_sk Certfile:/etc/hyperledger/fabric-ca-server-config/ca.org1.example.com-cert.pem Chainfile:ca-chain.pem} Signing:0xc4205a0e50 CSR:{CN:fabric-ca-server Names:[{C:US ST:North Carolina L: O:Hyperledger OU:Fabric SerialNumber:}] Hosts:[e3c7da43b062 localhost] KeyRequest:<nil> CA:0xc42059cea0 SerialNumber:} Registry:{MaxEnrollments:-1 Identities:[{ Name:**** Pass:**** Type:client Affiliation: MaxEnrollments:0 Attrs:map[hf.AffiliationMgr:1 firstName:* hf.Registrar.Roles:* hf.Registrar.DelegateRoles:* hf.Revoker:1 hf.IntermediateCA:1 hf.GenCRL:1 hf.Registrar.Attributes:firstName] }]} Affiliations:map[org2:[department1] org1:[department1 department2]] LDAP:{ Enabled:false URL:ldap://****:****#<host>:<port>/<base> UserFilter:(uid=%s) GroupFilter:(memberUid=%s) Attribute:{[uid member] [{ }] map[groups:[{ }]]} TLS:{false [] { }} } DB:{ Type:sqlite3 Datasource:fabric-ca-server.db TLS:{false [] { }} } CSP:0xc42059eba0 Client:<nil> Intermediate:{ParentServer:{ URL: CAName: } TLS:{Enabled:false CertFiles:[] Client:{KeyFile: CertFile:}} Enrollment:{ Name: Secret:**** CAName: AttrReqs:[] Profile: Label: CSR:<nil> Type:x509 }} CRL:{Expiry:24h0m0s} Idemix:{IssuerPublicKeyfile: IssuerSecretKeyfile: RevocationPublicKeyfile: RevocationPrivateKeyfile: RHPoolSize:1000 NonceExpiration:15s NonceSweepInterval:15m}}
2018/07/17 04:23:00 [DEBUG] CA Home Directory: /etc/hyperledger/fabric-ca-server/ca
2018/07/17 04:23:00 [DEBUG] Checking configuration file version '1.1.0' against server version: '1.2.0-stable'
2018/07/17 04:23:00 [DEBUG] Initializing BCCSP: &{ProviderName:SW SwOpts:0xc42059ec00 PluginOpts:<nil> Pkcs11Opts:<nil>}
2018/07/17 04:23:00 [DEBUG] Initializing BCCSP with software options &{SecLevel:256 HashFamily:SHA2 Ephemeral:false FileKeystore:0xc4205a1890 DummyKeystore:<nil>}
2018/07/17 04:23:00 [DEBUG] Initialize key material
2018/07/17 04:23:00 [DEBUG] Making CA filenames absolute
2018/07/17 04:23:00 [INFO] The CA key and certificate files already exist
2018/07/17 04:23:00 [INFO] Key file location: /etc/hyperledger/fabric-ca-server-config/3099498edb13f731965246b01e74514e4b80e978a058cf4cb9b99ab1408e2569_sk
2018/07/17 04:23:00 [INFO] Certificate file location: /etc/hyperledger/fabric-ca-server-config/ca.org1.example.com-cert.pem
2018/07/17 04:23:00 [DEBUG] Validating the CA certificate and key
2018/07/17 04:23:00 [DEBUG] Check CA certificate for valid dates
2018/07/17 04:23:00 [DEBUG] Check CA certificate for valid usages
2018/07/17 04:23:00 [DEBUG] Check CA certificate for valid IsCA value
2018/07/17 04:23:00 [DEBUG] Check that key type is supported
2018/07/17 04:23:00 [DEBUG] Check that key size is of appropriate length
2018/07/17 04:23:00 [DEBUG] Check that public key and private key match
2018/07/17 04:23:00 [DEBUG] Validation of CA certificate and key successful
2018/07/17 04:23:00 [DEBUG] Loading CN from existing enrollment information
2018/07/17 04:23:00 [DEBUG] Initializing DB
2018/07/17 04:23:00 [DEBUG] Initializing 'sqlite3' database at '/etc/hyperledger/fabric-ca-server/ca/fabric-ca-server.db'
2018/07/17 04:23:00 [DEBUG] Using sqlite database, connect to database in home (/etc/hyperledger/fabric-ca-server/ca/fabric-ca-server.db) directory
2018/07/17 04:23:00 [DEBUG] Creating SQLite database (/etc/hyperledger/fabric-ca-server/ca/fabric-ca-server.db) if it does not exist...
2018/07/17 04:23:00 [DEBUG] Creating users table if it does not exist
2018/07/17 04:23:00 [DEBUG] Creating affiliations table if it does not exist
2018/07/17 04:23:00 [DEBUG] Creating certificates table if it does not exist
2018/07/17 04:23:00 [DEBUG] Creating credentials table if it does not exist
2018/07/17 04:23:00 [DEBUG] Creating revocation_authority_info table if it does not exist
2018/07/17 04:23:00 [DEBUG] Creating nonces table if it does not exist
2018/07/17 04:23:01 [DEBUG] Creating properties table if it does not exist
2018/07/17 04:23:02 [DEBUG] Successfully opened sqlite3 DB
2018/07/17 04:23:02 [DEBUG] Checking database schema...
2018/07/17 04:23:02 [DEBUG] Update SQLite schema, if using outdated schema
2018/07/17 04:23:02 [DEBUG] Upgrade identities table
2018/07/17 04:23:02 [DEBUG] Creating users table if it does not exist
2018/07/17 04:23:03 [DEBUG] Upgrade affiliation table
2018/07/17 04:23:03 [DEBUG] Creating affiliations table if it does not exist
2018/07/17 04:23:04 [DEBUG] Upgrade certificates table
2018/07/17 04:23:04 [DEBUG] Creating certificates table if it does not exist
2018/07/17 04:23:05 [DEBUG] Initializing identity registry
2018/07/17 04:23:05 [DEBUG] Initialized DB identity registry
2018/07/17 04:23:05 [DEBUG] DB: Get properties [identity.level affiliation.level certificate.level]
2018/07/17 04:23:05 [DEBUG] Checking database levels 'map[affiliation.level:0 certificate.level:0 identity.level:0]' against server levels '&{Identity:1 Affiliation:1 Certificate:1 Credential:1 RAInfo:1 Nonce:1}'
2018/07/17 04:23:05 [DEBUG] Loading identity table
2018/07/17 04:23:05 [DEBUG] Loading identity 'admin'
2018/07/17 04:23:05 [DEBUG] DB: Getting identity admin
2018/07/17 04:23:05 [DEBUG] Max enrollment value verification - User specified max enrollment: 0, CA max enrollment: -1
2018/07/17 04:23:05 [DEBUG] DB: Add identity admin
2018/07/17 04:23:05 [DEBUG] Successfully added identity admin to the database
2018/07/17 04:23:05 [DEBUG] Registered identity: { Name:**** Pass:**** Type:client Affiliation: MaxEnrollments:-1 Attrs:map[hf.Revoker:1 hf.IntermediateCA:1 hf.GenCRL:1 hf.Registrar.Attributes:firstName hf.AffiliationMgr:1 firstName:* hf.Registrar.Roles:* hf.Registrar.DelegateRoles:*] }
2018/07/17 04:23:05 [DEBUG] Successfully loaded identity table
2018/07/17 04:23:05 [DEBUG] Loading affiliations table
2018/07/17 04:23:05 [DEBUG] DB: Add affiliation org2
2018/07/17 04:23:06 [DEBUG] Affiliation 'org2' added
2018/07/17 04:23:06 [DEBUG] DB: Add affiliation org2.department1
2018/07/17 04:23:07 [DEBUG] Affiliation 'org2.department1' added
2018/07/17 04:23:07 [DEBUG] DB: Add affiliation org1
2018/07/17 04:23:07 [DEBUG] Affiliation 'org1' added
2018/07/17 04:23:07 [DEBUG] DB: Add affiliation org1.department1
2018/07/17 04:23:08 [DEBUG] Affiliation 'org1.department1' added
2018/07/17 04:23:08 [DEBUG] DB: Add affiliation org1.department2
2018/07/17 04:23:08 [DEBUG] Affiliation 'org1.department2' added
2018/07/17 04:23:08 [DEBUG] Successfully loaded affiliations table
2018/07/17 04:23:08 [DEBUG] Checking and performing migration, if needed
2018/07/17 04:23:18 [DEBUG] Initializing enrollment signer
2018/07/17 04:23:18 [DEBUG] No key found in BCCSP keystore, attempting fallback
2018/07/17 04:23:18 [DEBUG] validating configuration
2018/07/17 04:23:18 [DEBUG] validate local profile
2018/07/17 04:23:18 [DEBUG] profile is valid
2018/07/17 04:23:18 [DEBUG] validate local profile
2018/07/17 04:23:18 [DEBUG] profile is valid
2018/07/17 04:23:18 [DEBUG] validate local profile
2018/07/17 04:23:18 [DEBUG] profile is valid
2018/07/17 04:23:18 [DEBUG] CA initialization successful
2018/07/17 04:23:18 [INFO] The issuer key was successfully stored. The public key is at: /etc/hyperledger/fabric-ca-server/IssuerPublicKey, secret key is at: /etc/hyperledger/fabric-ca-server/msp/keystore/IssuerSecretKey
2018/07/17 04:23:18 [INFO] The revocation key was successfully stored. The public key is at: /etc/hyperledger/fabric-ca-server/IssuerRevocationPublicKey, private key is at: /etc/hyperledger/fabric-ca-server/msp/keystore/IssuerRevocationPrivateKey
2018/07/17 04:23:18 [DEBUG] Comparing DNs from certificates: /etc/hyperledger/fabric-ca-server-config/ca.org1.example.com-cert.pem and /etc/hyperledger/fabric-ca-server/ca-cert.pem
2018/07/17 04:23:18 [DEBUG] Loading DNs from certificate /etc/hyperledger/fabric-ca-server-config/ca.org1.example.com-cert.pem
2018/07/17 04:23:18 [DEBUG] Loading DNs from certificate /etc/hyperledger/fabric-ca-server/ca-cert.pem
2018/07/17 04:23:18 [DEBUG] Check to see if two DNs are equal - &{issuer:CN=ca.org1.example.com,O=org1.example.com,L=San Francisco,ST=California,C=US subject:CN=ca.org1.example.com,O=org1.example.com,L=San Francisco,ST=California,C=US} and &{issuer:CN=example.com,OU=WWW,O=Internet Widgets\, Inc.,L=San Francisco,ST=California,C=US subject:CN=example.com,OU=WWW,O=Internet Widgets\, Inc.,L=San Francisco,ST=California,C=US}
2018/07/17 04:23:18 [INFO] Home directory for default CA: /etc/hyperledger/fabric-ca-server
2018/07/17 04:23:18 [DEBUG] 2 CA instance(s) running on server
2018/07/17 04:23:18 [INFO] Listening on http://0.0.0.0:7054
2018/07/17 04:38:12 [DEBUG] Cleaning up expired nonces for CA 'ca-org1'
2018/07/17 04:38:18 [DEBUG] Cleaning up expired nonces for CA ''
2018/07/17 04:46:57 [DEBUG] Received request for /api/v1/enroll
2018/07/17 04:46:57 [DEBUG] ca.Config: &{Version:1.1.0 Cfg:{Identities:{AllowRemove:false} Affiliations:{AllowRemove:false}} CA:{Name:ca-org1 Keyfile:/etc/hyperledger/fabric-ca-server-config/3099498edb13f731965246b01e74514e4b80e978a058cf4cb9b99ab1408e2569_sk Certfile:/etc/hyperledger/fabric-ca-server-config/ca.org1.example.com-cert.pem Chainfile:/etc/hyperledger/fabric-ca-server/ca/ca-chain.pem} Signing:0xc4205a0e50 CSR:{CN:ca.org1.example.com Names:[{C:US ST:North Carolina L: O:Hyperledger OU:Fabric SerialNumber:}] Hosts:[e3c7da43b062 localhost] KeyRequest:<nil> CA:0xc42059cea0 SerialNumber:} Registry:{MaxEnrollments:-1 Identities:[{ Name:**** Pass:**** Type:client Affiliation: MaxEnrollments:0 Attrs:map[hf.Registrar.DelegateRoles:* hf.Revoker:1 hf.IntermediateCA:1 hf.GenCRL:1 hf.Registrar.Attributes:firstName hf.AffiliationMgr:1 firstName:* hf.Registrar.Roles:*] }]} Affiliations:map[org2:[department1] org1:[department1 department2]] LDAP:{ Enabled:false URL:ldap://****:****#<host>:<port>/<base> UserFilter:(uid=%s) GroupFilter:(memberUid=%s) Attribute:{[uid member] [{ }] map[groups:[{ }]]} TLS:{false [] { }} } DB:{ Type:sqlite3 Datasource:/etc/hyperledger/fabric-ca-server/ca/fabric-ca-server.db TLS:{false [] { }} } CSP:0xc42059eba0 Client:<nil> Intermediate:{ParentServer:{ URL: CAName: } TLS:{Enabled:false CertFiles:[] Client:{KeyFile: CertFile:}} Enrollment:{ Name: Secret:**** CAName: AttrReqs:[] Profile: Label: CSR:<nil> Type:x509 }} CRL:{Expiry:24h0m0s} Idemix:{IssuerPublicKeyfile:/etc/hyperledger/fabric-ca-server/ca/IssuerPublicKey IssuerSecretKeyfile:/etc/hyperledger/fabric-ca-server/ca/msp/keystore/IssuerSecretKey RevocationPublicKeyfile:/etc/hyperledger/fabric-ca-server/ca/IssuerRevocationPublicKey RevocationPrivateKeyfile:/etc/hyperledger/fabric-ca-server/ca/msp/keystore/IssuerRevocationPrivateKey RHPoolSize:1000 NonceExpiration:15s NonceSweepInterval:15m}}
2018/07/17 04:46:57 [DEBUG] DB: Getting identity admin
2018/07/17 04:46:57 [DEBUG] DB: Login user admin with max enrollments of -1 and state of 0
2018/07/17 04:46:57 [DEBUG] DB: identity admin successfully logged in
2018/07/17 04:46:57 [DEBUG] DB: Getting identity admin
2018/07/17 04:46:57 [DEBUG] Processing sign request: id=admin, CommonName=admin, Subject=<nil>
2018/07/17 04:46:57 [DEBUG] Request is not for a CA signing certificate
2018/07/17 04:46:57 [DEBUG] Checking CSR fields to make sure that they do not exceed maximum character limits
2018/07/17 04:46:57 [DEBUG] Finished processing sign request
2018/07/17 04:46:57 [DEBUG] DB: Getting identity admin
2018-07-17T04:46:57.558556370Z 2018/07/17 04:46:57 [DEBUG] Attribute extension being added to certificate is: &{ID:[1 2 3 4 5 6 7 8 1] Critical:false Value:7b226174747273223a7b2266697273744e616d65223a222a222c2268662e5265676973747261722e41747472696275746573223a2266697273744e616d65227d7d}
2018/07/17 04:46:57 [DEBUG] Adding attribute extension to CSR: &{ID:[1 2 3 4 5 6 7 8 1] Critical:false Value:7b226174747273223a7b2266697273744e616d65223a222a222c2268662e5265676973747261722e41747472696275746573223a2266697273744e616d65227d7d}
2018/07/17 04:46:57 [INFO] signed certificate with serial number 624665886532998504441151201588115183399742601456
2018/07/17 04:46:57 [DEBUG] DB: Insert Certificate
2018/07/17 04:46:57 [DEBUG] Saved serial number as hex 6d6afcb3ed661e7f059b8a0b51829330fdfdf8f0
2018/07/17 04:46:57 [DEBUG] saved certificate with serial number 624665886532998504441151201588115183399742601456
2018/07/17 04:46:57 [DEBUG] Successfully incremented state for identity admin to 1
2018/07/17 04:46:57 [INFO] 172.18.0.1:47506 POST /api/v1/enroll 201 0 "OK"
2018/07/17 04:47:02 [DEBUG] Received request for /api/v1/register
2018/07/17 04:47:02 [DEBUG] Checking for revocation/expiration of certificate owned by 'admin'
2018/07/17 04:47:02 [DEBUG] DB: Get certificate by serial (6d6afcb3ed661e7f059b8a0b51829330fdfdf8f0) and aki (3099498edb13f731965246b01e74514e4b80e978a058cf4cb9b99ab1408e2569)
2018/07/17 04:47:02 [DEBUG] DB: Getting identity admin
2018/07/17 04:47:02 [DEBUG] Successful token authentication of 'admin'
2018/07/17 04:47:02 [DEBUG] Received registration request from admin: { Name:vijay Type:client Secret:**** MaxEnrollments:5 Affiliation:org1.department1 Attributes:[{firstName Lohith true}] CAName:ca-org1 }
2018/07/17 04:47:02 [DEBUG] canRegister - Check to see if user 'admin' can register
2018/07/17 04:47:02 [DEBUG] Checking to see if caller 'admin' can act on type 'client'
2018/07/17 04:47:02 [DEBUG] Checking to see if caller 'admin' is a registrar
2018/07/17 04:47:02 [DEBUG] Validating affiliation: org1.department1
2018/07/17 04:47:02 [DEBUG] Checking to see if affiliation 'org1.department1' contains caller's affiliation ''
2018/07/17 04:47:02 [DEBUG] Caller has root affiliation
2018/07/17 04:47:02 [DEBUG] DB: Get affiliation org1.department1
2018/07/17 04:47:02 [DEBUG] Checking to see if registrar can register the requested attributes: [{Name:firstName Value:Lohith ECert:true}]
2018/07/17 04:47:02 [DEBUG] Validating that registrar with the following values for hf.Registrar.Attributes 'firstName' is authorized to register the requested attribute '&{Name:firstName Value:Lohith ECert:true}'
2018/07/17 04:47:02 [DEBUG] Checking if registrar can register attribute: firstName
2018/07/17 04:47:02 [DEBUG] Performing authorization check...
2018/07/17 04:47:02 [DEBUG] Registering user id: vijay
2018/07/17 04:47:02 [DEBUG] Max enrollment value verification - User specified max enrollment: 5, CA max enrollment: -1
2018/07/17 04:47:02 [DEBUG] DB: Getting identity vijay
2018/07/17 04:47:02 [DEBUG] DB: Add identity vijay
2018/07/17 04:47:02 [DEBUG] Successfully added identity vijay to the database
2018/07/17 04:47:02 [INFO] 172.18.0.1:47512 POST /api/v1/register 201 0 "OK"
2018/07/17 04:47:02 [DEBUG] Received request for /api/v1/enroll
2018/07/17 04:47:02 [DEBUG] ca.Config: &{Version:1.1.0 Cfg:{Identities:{AllowRemove:false} Affiliations:{AllowRemove:false}} CA:{Name:ca-org1 Keyfile:/etc/hyperledger/fabric-ca-server-config/3099498edb13f731965246b01e74514e4b80e978a058cf4cb9b99ab1408e2569_sk Certfile:/etc/hyperledger/fabric-ca-server-config/ca.org1.example.com-cert.pem Chainfile:/etc/hyperledger/fabric-ca-server/ca/ca-chain.pem} Signing:0xc4205a0e50 CSR:{CN:ca.org1.example.com Names:[{C:US ST:North Carolina L: O:Hyperledger OU:Fabric SerialNumber:}] Hosts:[e3c7da43b062 localhost] KeyRequest:<nil> CA:0xc42059cea0 SerialNumber:} Registry:{MaxEnrollments:-1 Identities:[{ Name:**** Pass:**** Type:client Affiliation: MaxEnrollments:0 Attrs:map[hf.GenCRL:1 hf.Registrar.Attributes:firstName hf.AffiliationMgr:1 firstName:* hf.Registrar.Roles:* hf.Registrar.DelegateRoles:* hf.Revoker:1 hf.IntermediateCA:1] }]} Affiliations:map[org2:[department1] org1:[department1 department2]] LDAP:{ Enabled:false URL:ldap://****:****#<host>:<port>/<base> UserFilter:(uid=%s) GroupFilter:(memberUid=%s) Attribute:{[uid member] [{ }] map[groups:[{ }]]} TLS:{false [] { }} } DB:{ Type:sqlite3 Datasource:/etc/hyperledger/fabric-ca-server/ca/fabric-ca-server.db TLS:{false [] { }} } CSP:0xc42059eba0 Client:<nil> Intermediate:{ParentServer:{ URL: CAName: } TLS:{Enabled:false CertFiles:[] Client:{KeyFile: CertFile:}} Enrollment:{ Name: Secret:**** CAName: AttrReqs:[] Profile: Label: CSR:<nil> Type:x509 }} CRL:{Expiry:24h0m0s} Idemix:{IssuerPublicKeyfile:/etc/hyperledger/fabric-ca-server/ca/IssuerPublicKey IssuerSecretKeyfile:/etc/hyperledger/fabric-ca-server/ca/msp/keystore/IssuerSecretKey RevocationPublicKeyfile:/etc/hyperledger/fabric-ca-server/ca/IssuerRevocationPublicKey RevocationPrivateKeyfile:/etc/hyperledger/fabric-ca-server/ca/msp/keystore/IssuerRevocationPrivateKey RHPoolSize:1000 NonceExpiration:15s NonceSweepInterval:15m}}
2018/07/17 04:47:02 [DEBUG] DB: Getting identity vijay
2018/07/17 04:47:02 [DEBUG] DB: Login user vijay with max enrollments of 5 and state of 0
2018/07/17 04:47:02 [DEBUG] DB: identity vijay successfully logged in
2018/07/17 04:47:02 [DEBUG] DB: Getting identity vijay
2018/07/17 04:47:02 [DEBUG] Processing sign request: id=vijay, CommonName=vijay, Subject=<nil>
2018/07/17 04:47:02 [DEBUG] Request is not for a CA signing certificate
2018/07/17 04:47:02 [DEBUG] Checking CSR fields to make sure that they do not exceed maximum character limits
2018/07/17 04:47:02 [DEBUG] Finished processing sign request
2018/07/17 04:47:02 [DEBUG] DB: Getting identity vijay
2018/07/17 04:47:02 [DEBUG] Attribute extension being added to certificate is: &{ID:[1 2 3 4 5 6 7 8 1] Critical:false Value:7b226174747273223a7b2266697273744e616d65223a224c6f68697468227d7d}
2018/07/17 04:47:02 [DEBUG] Adding attribute extension to CSR: &{ID:[1 2 3 4 5 6 7 8 1] Critical:false Value:7b226174747273223a7b2266697273744e616d65223a224c6f68697468227d7d}
2018/07/17 04:47:02 [INFO] signed certificate with serial number 124540954052671924407683868974981960232864106719
2018/07/17 04:47:02 [DEBUG] DB: Insert Certificate
2018/07/17 04:47:02 [DEBUG] Saved serial number as hex 15d09c09d65b92d0f28dc7287fdd01d52764c8df
2018/07/17 04:47:02 [DEBUG] saved certificate with serial number 124540954052671924407683868974981960232864106719
2018/07/17 04:47:02 [DEBUG] Successfully incremented state for identity vijay to 1
2018/07/17 04:47:02 [INFO] 172.18.0.1:47516 POST /api/v1/enroll 201 0 "OK"
I suggest trying the following:
Adding more conditional statements in order to evaluate what is going wrong with your application, such as something like:
func (t* SimpleChaincode) get_attribute(stub shim.ChaincodeStubInterface, args []string) pb.Response {
var attribute_name string
if len(args) != 1 {
return shim.Error("Incorrect number of arguments. Expecting only attribute name")
}
attribute_name = args[0]
val, ok, err := cid.GetAttributeValue(stub, attribute_name)
if err != nil {
return shim.Error("Error retriving attribute " + attribute_name + ", error: " + err.Error())
}
if !ok {
return shim.Error("User does not have attribute " + attribute_name)
}
return shim.Success([]byte(val))
}
Reviewing an example of how to query chaincode:
var Fabric_Client = require('fabric-client');
var path = require('path');
var util = require('util');
var os = require('os');
//
var fabric_client = new Fabric_Client();
var attribute = ''
var user = ''
if ((process.argv).length > 3) {
user = process.argv[2]
attribute = process.argv[3]
} else {
console.log("Usage: node " + process.argv[1] + " username attribute")
process.exit(1)
}
// setup the fabric network
var channel = fabric_client.newChannel('mychannel');
var peer = fabric_client.newPeer('grpc://peer0.org1.example.com:7051');
channel.addPeer(peer);
//
var member_user = null;
var store_path = path.join(__dirname, 'hfc-key-store');
console.log('Store path:'+store_path);
var tx_id = null;
// create the key value store as defined in the fabric-client/config/default.json 'key-value-store' setting
Fabric_Client.newDefaultKeyValueStore({ path: store_path
}).then((state_store) => {
// assign the store to the fabric client
fabric_client.setStateStore(state_store);
var crypto_suite = Fabric_Client.newCryptoSuite();
// use the same location for the state store (where the users' certificate are kept)
// and the crypto store (where the users' keys are kept)
var crypto_store = Fabric_Client.newCryptoKeyStore({path: store_path});
crypto_suite.setCryptoKeyStore(crypto_store);
fabric_client.setCryptoSuite(crypto_suite);
// get the enrolled user from persistence, this user will sign all requests
return fabric_client.getUserContext(user, true);
}).then((user_from_store) => {
if (user_from_store && user_from_store.isEnrolled()) {
console.log('Successfully loaded ' + user + ' from persistence');
member_user = user_from_store;
} else {
throw new Error('Failed to get ' + user + ' run registerUser.js');
}
// queryCar chaincode function - requires 1 argument, ex: args: ['CAR4'],
// queryAllCars chaincode function - requires no arguments , ex: args: [''],
const request = {
//targets : --- letting this default to the peers assigned to the channel
chaincodeId: 'acl',
fcn: 'get_attribute',
args: [attribute]
};
// send the query proposal to the peer
return channel.queryByChaincode(request);
}).then((query_responses) => {
console.log("Query has completed, checking results");
// query_responses could have more than one results if there multiple peers were used as targets
if (query_responses && query_responses.length == 1) {
if (query_responses[0] instanceof Error) {
console.error("error from query = ", query_responses[0]);
} else {
console.log("Response is ", query_responses[0].toString());
}
} else {
console.log("No payloads were returned from query");
}
}).catch((err) => {
console.error('Failed to query successfully :: ' + err);
});
Source URL for code: https://gist.github.com/tboloo/e06e2a86f99fc8e7fca295daf4d3f34c
Implementing ABAC
It wasn't clear to me how you are enforcing the decision making for your ABAC with Hyperledger Fabric. My understanding is that Hyperledger Fabric doesn't come with a built in architecture to handle this (PDP, etc.). I suggest using an implementation of XACML to externally handle dynamic authorization.
If you are using
var tx_id = client.newTransactionID(true);
try setting it to
var tx_id = client.newTransactionID();
I am trying to implement Node.js SDK for my Hyperledger Fabric business network. My network has one organisation, four peers, one channel, solo ordered and one CA.
I have successful managed to issue the JWT identity for user Jim and org1 organization.
When I try to invoke the the chaincode using:
curl -s -X POST \
http://localhost:4000/channels/c1/chaincodes/txman \
-H "authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1MTAwODIyNzMsInVzZXJuYW1lIjoiSmltIiwib3JnTmFtZSI6Im9yZzEiLCJpYXQiOjE1MTAwNDYyNzN9.NTDYky6jj_y1COP163Z_HA6q4v7S1_jN6BxL00Bo-Xo" \
-H "content-type: application/json" \
-d '{
"fcn":"queryTx",
"args":["TX0"]
}'
I am getting the following error:
2017-11-07 11:18:34.286] [DEBUG] SampleWebApp - Decoded from JWT token: username - Jim, orgname - org1
[2017-11-07 11:18:34.287] [DEBUG] SampleWebApp - ==================== INVOKE ON CHAINCODE ==================
[2017-11-07 11:18:34.287] [DEBUG] SampleWebApp - USERNAME: Jim
[2017-11-07 11:18:34.287] [DEBUG] SampleWebApp - ORGNAME: org1
[2017-11-07 11:18:34.287] [DEBUG] SampleWebApp - channelName : c1
[2017-11-07 11:18:34.288] [DEBUG] SampleWebApp - chaincodeName : txman
[2017-11-07 11:18:34.288] [DEBUG] SampleWebApp - fcn : queryTx
[2017-11-07 11:18:34.288] [DEBUG] SampleWebApp - args : TX0
[2017-11-07 11:18:34.292] [DEBUG] invoke-chaincode -
============ invoke transaction on organization org1 ============
[2017-11-07 11:18:34.292] [DEBUG] Helper - [FileKeyValueStore.js]: FileKeyValueStore.js - constructor
[2017-11-07 11:18:34.293] [DEBUG] Helper - [FileKeyValueStore.js]: FileKeyValueStore -- getValue
[2017-11-07 11:18:34.298] [DEBUG] Helper - [crypto_ecdsa_aes]: importKey - start
[2017-11-07 11:18:34.298] [DEBUG] Helper - [crypto_ecdsa_aes]: importKey - have the key [Circular]
[2017-11-07 11:18:34.300] [DEBUG] Helper - [utils.CryptoKeyStore]: _getKeyStore resolving store
[2017-11-07 11:18:34.300] [DEBUG] Helper - [ecdsa/key.js]: ECDSA curve param X: c3cd1a318e957b0200852e4ed6c9601eadc8225eb9f79acc11b1c65df142446a
[2017-11-07 11:18:34.300] [DEBUG] Helper - [ecdsa/key.js]: ECDSA curve param Y: f7bbe9fd43eaae1a0a1d2eb3024ecd79bd63c39a8d1b3b978bc8b4a652d0ef63
[2017-11-07 11:18:34.301] [DEBUG] Helper - [FileKeyValueStore.js]: FileKeyValueStore -- setValue
[2017-11-07 11:18:34.303] [DEBUG] Helper - [utils.CryptoKeyStore]: _getKeyStore resolving store
[2017-11-07 11:18:34.303] [DEBUG] Helper - [FileKeyValueStore.js]: FileKeyValueStore -- getValue
[2017-11-07 11:18:34.304] [DEBUG] Helper - [ecdsa/key.js]: ECDSA curve param X: c3cd1a318e957b0200852e4ed6c9601eadc8225eb9f79acc11b1c65df142446a
[2017-11-07 11:18:34.304] [DEBUG] Helper - [ecdsa/key.js]: ECDSA curve param Y: f7bbe9fd43eaae1a0a1d2eb3024ecd79bd63c39a8d1b3b978bc8b4a652d0ef63
[2017-11-07 11:18:34.305] [DEBUG] Helper - [ecdsa/key.js]: ECDSA curve param X: c3cd1a318e957b0200852e4ed6c9601eadc8225eb9f79acc11b1c65df142446a
[2017-11-07 11:18:34.306] [DEBUG] Helper - [ecdsa/key.js]: ECDSA curve param Y: f7bbe9fd43eaae1a0a1d2eb3024ecd79bd63c39a8d1b3b978bc8b4a652d0ef63
[2017-11-07 11:18:34.308] [DEBUG] Helper - [ecdsa/key.js]: ECDSA curve param X: c3cd1a318e957b0200852e4ed6c9601eadc8225eb9f79acc11b1c65df142446a
[2017-11-07 11:18:34.308] [DEBUG] Helper - [ecdsa/key.js]: ECDSA curve param Y: f7bbe9fd43eaae1a0a1d2eb3024ecd79bd63c39a8d1b3b978bc8b4a652d0ef63
[2017-11-07 11:18:34.309] [DEBUG] Helper - [FileKeyValueStore.js]: FileKeyValueStore -- setValue
[2017-11-07 11:18:34.309] [INFO] Helper - Successfully loaded member from persistence
[2017-11-07 11:18:34.313] [DEBUG] invoke-chaincode - Sending transaction "{"_nonce":{"type":"Buffer","data":[218,130,209,154,87,123,73,36,118,155,194,91,27,250,14,216,117,202,164,5,16,237,96,208]},"_transaction_id":"dba107be47745c51eb75b6a927d68f0afd58b325c5e36326b40a729f9d858f43"}"
[2017-11-07 11:18:34.341] [DEBUG] Helper - [crypto_ecdsa_aes]: ecdsa signature: Signature {
r: <BN: cf5e0aaef0fd10d9abf468e025c6f6c1e8991d0dcddf5f5aa6b285ebdcd920ad>,
s: <BN: 8de356dcfda3ba3fb0d6decfc0e43995b3297bc7037fd3997bfba561e6d626d>,
recoveryParam: 0 }
error: [client-utils.js]: sendPeersProposal - Promise is rejected: Error: Failed to deserialize creator identity, err The supplied identity is not valid, Verify() returned x509: certificate signed by unknown authority (possibly because of "x509: ECDSA verification failure" while trying to verify candidate authority certificate "ca.org1.example.com")
at /home/brankoterzic/Desktop/devGC/txman/node_modules/grpc/src/client.js:554:15
error: [client-utils.js]: sendPeersProposal - Promise is rejected: Error: Failed to deserialize creator identity, err The supplied identity is not valid, Verify() returned x509: certificate signed by unknown authority (possibly because of "x509: ECDSA verification failure" while trying to verify candidate authority certificate "ca.org1.example.com")
at /home/brankoterzic/Desktop/devGC/txman/node_modules/grpc/src/client.js:554:15
error: [client-utils.js]: sendPeersProposal - Promise is rejected: Error: Failed to deserialize creator identity, err The supplied identity is not valid, Verify() returned x509: certificate signed by unknown authority (possibly because of "x509: ECDSA verification failure" while trying to verify candidate authority certificate "ca.org1.example.com")
at /home/brankoterzic/Desktop/devGC/txman/node_modules/grpc/src/client.js:554:15
error: [client-utils.js]: sendPeersProposal - Promise is rejected: Error: Failed to deserialize creator identity, err The supplied identity is not valid, Verify() returned x509: certificate signed by unknown authority (possibly because of "x509: ECDSA verification failure" while trying to verify candidate authority certificate "ca.org1.example.com")
at /home/brankoterzic/Desktop/devGC/txman/node_modules/grpc/src/client.js:554:15
error: [client-utils.js]: sendPeersProposal - Promise is rejected: Error: Failed to deserialize creator identity, err The supplied identity is not valid, Verify() returned x509: certificate signed by unknown authority (possibly because of "x509: ECDSA verification failure" while trying to verify candidate authority certificate "ca.org1.example.com")
at /home/brankoterzic/Desktop/devGC/txman/node_modules/grpc/src/client.js:554:15
[2017-11-07 11:18:34.400] [ERROR] invoke-chaincode - transaction proposal was bad
[2017-11-07 11:18:34.401] [ERROR] invoke-chaincode - transaction proposal was bad
[2017-11-07 11:18:34.401] [ERROR] invoke-chaincode - transaction proposal was bad
[2017-11-07 11:18:34.401] [ERROR] invoke-chaincode - transaction proposal was bad
[2017-11-07 11:18:34.401] [ERROR] invoke-chaincode - transaction proposal was bad
[2017-11-07 11:18:34.401] [ERROR] invoke-chaincode - Failed to send Proposal or receive valid response. Response null or status is not 200. exiting...
[2017-11-07 11:18:34.401] [ERROR] invoke-chaincode - Failed to order the transaction. Error code: undefined
I was following the balance-transfer Node.js example: https://github.com/hyperledger/fabric-samples/tree/release/balance-transfer
Thank you for your answers!
I have solved this by setting up the generated crypto for CA
crypto-config/peerOrganizations/org1.example.com/ca/361a9254173cb1a97c057432094a20db9578ca69a152725f96323e00379c5ad6_sk
to
FABRIC_CA_SERVER_CA_KEYFILE=/etc/hyperledger/fabric-ca-server-config/361a9254173cb1a97c057432094a20db9578ca69a152725f96323e00379c5ad6_sk
in docker-compose-base.yaml env variable.
We are developing an app using sails.js.
In this app we have an upload controller:
https://github.com/makinacorpus/mnhn_bai/blob/master/api/controllers/Object3DController.js
This controller use skipper under the hood, as explained in the documentation.
Now the problem is that when we upload big files, they are incompletely stored, the uploaded size is never the same and varies from 7mb to 14mb for a 15 mb file.
The architecture is as follow:
haproxy -> nginx -> node.js/sails.
If we replace the nginx reverse proxy by a simple apache + proxypass configuration, the uploads work flawlessly.
If we replace the node.js app by a simple python upload controller (in flask, eg) the upload is also showing the correct length and data.
Of course nginx has been correctly configured for the buffer sizes, client_body_timeout, and client_max_body_size, and as i said, the flask have is just receiving correctly the upload.
The upload with the nginx app result in a 200 response, so it seems the file was uploaded, but in fact, on the disk, the file is incomplete.
On the nginx debug log we can see that:
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header:
"POST /admin/edit_object/6 HTTP/1.1^M
Host: xxxxxx.makina-corpus.net^M
X-Real-IP: xxxx^M
X-Forwarded-For: xxxxx^M
X-NginX-Proxy: true^M
X-Forwarded-Proto: http^M
Connection: upgrade^M
Content-Length: 15361775^M
Origin: http://bai.makina-corpus.net^M
User-Agent: Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) CasperJS/1.1.0-beta3+PhantomJS/1.9.8 Safari/534.34^M
Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryRt4v4f7RkrlzUEX2^M
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8^M
Referer: http://xxxxxxxxxx.makina-corpus.net/admin/edit_object/6^M
Cookie: sails.sid=s%3Akv_Gxxxxxxxx2F5iaDWA^M
Accept-Encoding: gzip^M
Accept-Language: en,*^M
Authorization: Basic xxxx=^M
^M
"
2014/12/03 01:57:23 [debug] 39583#0: *1 http cleanup add: 00000000011CC520
2014/12/03 01:57:23 [debug] 39583#0: *1 init keepalive peer
2014/12/03 01:57:23 [debug] 39583#0: *1 get keepalive peer
2014/12/03 01:57:23 [debug] 39583#0: *1 get rr peer, try: 1
2014/12/03 01:57:23 [debug] 39583#0: *1 get keepalive peer: using connection 0000000001156018
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream connect: -4
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream send request
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer buf fl:0 s:806
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer buf fl:1 s:15361775
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer in: 00000000011CC5C0
2014/12/03 01:57:23 [debug] 39583#0: *1 tcp_nopush
2014/12/03 01:57:23 [debug] 39583#0: *1 writev: 806
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: #0 15361775
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: 2776864, #0 2776864:15361775
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer out: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer add: 35: 60000:1417568303245
2014/12/03 01:57:23 [debug] 39583#0: *1 http run request: "/admin/edit_object/6?"
2014/12/03 01:57:23 [debug] 39583#0: *1 http request empty handler
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream request: "/admin/edit_object/6?"
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream send request handler
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream send request
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer in: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: #2776864 12584911
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: 2488810, #2776864 2488810:12584911
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer out: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer del: 35: 1417568303245
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer add: 35: 60000:1417568303254
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream request: "/admin/edit_object/6?"
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream process header
2014/12/03 01:57:23 [debug] 39583#0: *1 malloc: 00000000011CD000:262144
2014/12/03 01:57:23 [debug] 39583#0: *1 recv: fd:35 369 of 262144
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy status 200 "200 OK"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "X-Powered-By: Sails <sailsjs.org>"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Access-Control-Allow-Origin: "
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Access-Control-Allow-Credentials: "
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Access-Control-Allow-Methods: "
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Access-Control-Allow-Headers: "
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Content-Type: application/json; charset=utf-8"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Content-Length: 33"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Vary: Accept-Encoding"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Date: Wed, 03 Dec 2014 00:57:23 GMT"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "Connection: keep-alive"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header done
2014/12/03 01:57:23 [debug] 39583#0: *1 uploadprogress error-tracker error: 0
2014/12/03 01:57:23 [debug] 39583#0: *1 xslt filter header
2014/12/03 01:57:23 [debug] 39583#0: *1 HTTP/1.1 200 OK^M
Server: nginx^M
Date: Wed, 03 Dec 2014 00:57:23 GMT^M
The problem seems that skipper never seems to hit the 'finish' event at the upstream level, infinite loop ?
The sails stdout
Parser: Done reading textparam through field `gallery`
Parser: Done reading textparam through field `category`
Parser: Done reading textparam through field `copyright`
Parser: Done reading textparam through field `published`
Parser: Done reading textparam through field `filename_3D`
Parser: Done reading textparam through field `filename_flat`
Parser: Done reading textparam through field `preview`
Parser: Done reading textparam through field `preview_animated`
Something is trying to read from Upstream `media_files`...
Passing control to app...
User allowed : admin ( 1 )
RenamerPump:
• dirname => undefined
• field => media_files
• fd => 04cb80ba-dce6-4a1d-9b54-ac8b08ca3e06
100
100
100
100
100
100
100
100
100
100
100
The interesting thing is that the file on the disk after upload contains at the end the headers from another unrelated request:
07bb890: 3130 3130 3130 3130 3130 3130 3130 3130 1010101010101010
07bb8a0: 3130 3130 3130 3130 3130 3130 3130 3130 1010101010101010
07bb8b0: 3130 3130 3130 3130 3130 3130 3130 3130 1010101010101010
07bb8c0: 3130 3130 3130 3130 3130 3130 4745 5420 101010101010GET
07bb8d0: 2f20 4854 5450 2f31 2e31 0d0a 486f 7374 / HTTP/1.1..Host
07bb8e0: xxxx xxxx xxxx xxxx xxxx xxxx 2d63 6f72 : xxx.makina-cor
07bb8f0: 7075 732e 6e65 740d 0a58 2d52 6561 6c2d pus.net..X-Real-
07bb900: 4950
07bb910: 3134 0d0a 582d 466f 7277 6172 6465 642d 14..X-Forwarded-
07bb920: 466f For: xxxxxxxxxxx
07bb930: 2e31
07bb940: 2e31 340d 0a58 2d4e 6769 6e58 2d50 726f .14..X-NginX-Pro
07bb950: 7879 3a20 7472 7565 0d0a 582d 466f 7277 xy: true..X-Forw
07bb960: 6172 6465 642d 5072 6f74 6f3a 2068 7474 arded-Proto: htt
07bb970: 700d 0a43 6f6e 6e65 6374 696f 6e3a 2075 p..Connection: u
07bb980: 7067 7261 6465 0d0a 5573 6572 2d41 6765 pgrade..User-Age
07bb990: 6e74 3a20 4d6f 7a69 6c6c 612f 352e 3020 nt: Mozilla/5.0
07bb9a0: 2855 6e6b 6e6f 776e 3b20 4c69 6e75 7820 (Unknown; Linux
07bb9b0: 7838 365f 3634 2920 4170 706c 6557 6562 x86_64) AppleWeb
07bb9c0: 4b69 742f 3533 342e 3334 2028 4b48 544d Kit/534.34 (KHTM
07bb9d0: 4c2c 206c 696b 6520 4765 636b 6f29 2043 L, like Gecko) C
07bb9e0: 6173 7065 724a 532f 312e 312e 302d 6265 asperJS/1.1.0-be
07bb9f0: 7461 332b 5068 616e 746f 6d4a 532f 312e ta3+PhantomJS/1.
07bba00: 392e 3820 5361 6661 7269 2f35 3334 2e33 9.8 Safari/534.3
07bba10: 340d 0a41 6363 6570 743a 2074 6578 742f 4..Accept: text/
07bba20: 6874 6d6c 2c61 7070 6c69 6361 7469 6f6e html,application
07bba30: 2f78 6874 6d6c 2b78 6d6c 2c61 7070 6c69 /xhtml+xml,appli
07bba40: 6361 7469 6f6e 2f78 6d6c 3b71 3d30 2e39 cation/xml;q=0.9
07bba50: 2c2a 2f2a 3b71 3d30 2e38 0d0a 4163 6365 ,*/*;q=0.8..Acce
07bba60: 7074 2d45 6e63 6f64 696e 673a 2067 7a69 pt-Encoding: gzi
07bba70: 700d 0a41 6363 6570 742d 4c61 6e67 7561 p..Accept-Langua
07bba80: 6765 3a20 656e 2c2a 0d0a 4175 7468 6f72 ge: en,*..Author
07bba90: 697a 6174 696f 6e3a 2042 6173 6963 2063 ization: Basic c
07bbaa0: 6d39 xxxx xxxx xxxx xxxx 3d0d 0a xxxx=..
(END)
And in other requests, we have not some other requests headers, but just an incomplete file.
Here, the missing bits are from the end of the original file, the start is always correct.
Note that the main difference with apache is that nginx is sending data is quick big bursts to the sails app. On the contrary apache is streaming the request.
This is because nginx does request buffering.
If someone has an idea from where to continue in skipper to dig out that upload problem !
If i replace the save method by this example, i see that the bits coming from nginx are written correctly, i have the full and correct file in the POSTed data, so the error is clearly somewhere in skipper request consumption
var body = "";
req.on('data', function (chunk) {
body += chunk;
});
req.on('end', function () {
console.log('POSTed: ' + body.length);
console.log('POSTed: ' + body.slice(-400));
res.writeHead(200);
res.end('<html/>');
});
From the nginx debug log it seems that the problem is due to early return of response from the backend - note that in the last sendfile() call nginx was able to send only 2488810 out of 12584911 bytes it tried to:
...
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer in: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: #2776864 12584911
2014/12/03 01:57:23 [debug] 39583#0: *1 sendfile: 2488810, #2776864 2488810:12584911
2014/12/03 01:57:23 [debug] 39583#0: *1 chain writer out: 00000000011CC5D0
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer del: 35: 1417568303245
2014/12/03 01:57:23 [debug] 39583#0: *1 event timer add: 35: 60000:1417568303254
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream request: "/admin/edit_object/6?"
2014/12/03 01:57:23 [debug] 39583#0: *1 http upstream process header
2014/12/03 01:57:23 [debug] 39583#0: *1 malloc: 00000000011CD000:262144
2014/12/03 01:57:23 [debug] 39583#0: *1 recv: fd:35 369 of 262144
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy status 200 "200 OK"
2014/12/03 01:57:23 [debug] 39583#0: *1 http proxy header: "X-Powered-By: Sails <sailsjs.org>"
...
And the backend returned an 200 OK answer. At this point nginx thinks that there is no reason to send the rest of the request body and stops sending it - this what causes incomplete uploaded files. Additionally, you have keepalive upstream connections configured, and you are hitting this bug - and this is why you see headers of an unrelated request.
Teaching the backend code to only send a response after the request is fully read, as in your test code, should resolve the problem.
So, the solution i found is to hack a bodyparser which use formidable.
No more problem :).
For the record, it was a bit of a hack to switch the bodyparser in the middlewares:
config/http.js
module.exports.http = {
middleware: {
bodyParser: false,
cbodyParser: require('../bodyParser')(
{urls: [/\/admin\/edit_object/]}),
order: [
'startRequestTimer',
'cookieParser',
'session',
'cbodyParser',
'handleBodyParserError',
'compress',
'methodOverride',
'poweredBy',
'$custom',
'router',
'www',
'favicon',
'404',
'500'
],
}
};
bodyparser.js:
/**
* Module dependencies
// Configure body parser components
*/
var _ = require('lodash');
var util = require('util');
var formidable = require('formidable');
function mime(req) {
var str = req.headers['content-type'] || '';
return str.split(';')[0];
}
function parseMultipart(req, res, next) {
req.form = new formidable.IncomingForm();
req.form.uploadDir = sails.config.data.__uploadData;
req.form.maxFieldsSize = sails.config.maxsize;
req.form.multiple = true;
// res.setTimeout(0);
req.form.parse(req, function(err, fields, files) {
if (err)
return next(err);
else {
req.files = files;
req.fields = fields;
req.body = extend(fields, files);
next();
}
});
}
function extend(target) {
var key, obj;
for (var i = 1, l = arguments.length; i < l; i++) {
if ((obj = arguments[i])) {
for (key in obj)
target[key] = obj[key];
}
}
return target;
}
function disable_parser(opts, req, res) {
var matched = false;
try {
var method = null;
try {method = req.method.toLowerCase();}
catch(err){ /* */}
if(method) {
_(opts.urls).forEach(function(turl) {
if (method === 'post' && req.url.match(turl)) {
// console.log("matched"+ req.url);
if(!matched) matched = true;
};});
}
} catch(err) { debug(err);/* pass */ }
return matched;
}
module.exports = function toParseHTTPBody(options) {
options = options || {};
var bodyparser = require('skipper')(options);
// NAME of anynonymous func IS IMPORTANT (same as the middleware in config) !!!
return function cbodyParser(req, res, next) {
var err_hdler = function(err) {};
if (disable_parser(options, req, res) && mime(req) == 'multipart/form-data') {
return parseMultipart(req, res, next);
} else {
return bodyparser(req, res, next);
}
};
};
Indeed, sails let to think that we can override the bodyParser, but we cant as it will result in an anonymous function but the express router only map "named" function...
We faced a similar issue. I dont know if our solution will work for you or not, but here goes.
For very large files, the csrf gets left out of request packet. So we need to send the csrf in request header rather than request body. For that we changed the XMLHttpRequest a little bit.
/*
Putting csrf in Header as some large
files need this mechanism to upload
*/
(function() {
var send = XMLHttpRequest.prototype.send,
token = csrfToken; //csrfToken is global
XMLHttpRequest.prototype.send = function(data) {
this.setRequestHeader('X-CSRF-Token', token);
return send.apply(this, arguments);
};
}());
From now, every request will have csrf in the header. This solved the problem for us. Hope this helps you too.