Error for GPU prediction

I am running into couple of errors with inference or prediction performance on GPU using AlexNet CNN.  This is the code
"""
Usage:
python alexnet_pretrain.py [batch size] [number of GPUs]
"""
CAFFE2_ROOT = '/local/caffe2/caffe2'
CAFFE_MODELS = '/local/caffe2/build/caffe2/python/models'
from caffe2.proto import caffe2_pb2
import numpy as np
import skimage.io
import skimage.transform
from matplotlib import pyplot
import os
from caffe2.python import core, workspace
import urllib2
import time
import math
import caffe2.python._import_c_extension as C
import sys
print 'Required modules imported.'
##change path to location of cifar10Test folder
IMAGE_FOLDER =  '/local/Deep_Learning_Benchmarking/cifar10Test'
BATCH_SIZE = int(sys.argv[1])
print 'BATCH SIZE: ', BATCH_SIZE
BENCHMARKING = True
NUM_GPUS = int(sys.argv[2])
imgFileList = []
##puts images from 1 category in a list
category = 'automobile'
for imgFile in os.listdir(os.path.join(IMAGE_FOLDER, category)):
imgFileList.append(os.path.join(IMAGE_FOLDER, category, imgFile))
MODEL = 'bvlc_alexnet', 'exec_net.pb', 'predict_net.pb', 'ilsvrc_2012_mean.npy', 227
##codes for image ID
codes =  "https://gist.githubusercontent.com/aaronmarkham/cd3a6b6ac071eca6f7b4a6e40e6038aa/raw/9edb4038a37da6b5a44c3b5bc52e448ff09bfe5b/alexnet_codes"
print "Config set!"
##crop out the edges of the image
def crop_center(img, cropx, cropy):
y,x,c = img.shape
startx = x//2-(cropx//2)
starty = y//2-(cropy//2)
return img[starty:starty+cropy, startx:startx+cropx]
##rescale the image to the given height and width
def rescale(img, input_height, input_width):
##print "original image shape: " +str(img.shape) + ' and remember it should be in h,w,c!'
##print "model's input shape is %dx%d" % (input_height, input_width)
aspect = img.shape[1]/float(img.shape[0])
##print 'original aspect ratio: ' + str(aspect)
if aspect > 1:
    # landscape image
    res = int(aspect*input_height)
    imgScaled = skimage.transform.resize(img, (input_width,res))
if aspect < 1:
    # portrait image
    res = int(input_width/aspect)
    imgScaled = skimage.tansform.resize(img, (res,input_height))
if aspect == 1:
    ##square image
    imgScaled = skimage.transform.resize(img, (input_width, input_height))

return imgScaled

print 'Functions set.'
CAFFE2_ROOT = os.path.expanduser(CAFFE2_ROOT)
CAFFE_MODELS = os.path.expanduser(CAFFE_MODELS)
##search for necessary files
MEAN_FILE = os.path.join(CAFFE_MODELS, MODEL[0], MODEL[3])
if not os.path.exists(MEAN_FILE):
mean = 128
else:
mean = np.load(MEAN_FILE).mean(1).mean(1)
mean = mean[:, np.newaxis, np.newaxis]
##mean = 128
print 'Mean was set to: ', mean
INPUT_IMAGE_SIZE = MODEL[4]
if not os.path.exists(CAFFE2_ROOT):
print "CAFFE2_ROOT doesn't exist!"
INIT_NET = os.path.join(CAFFE_MODELS, MODEL[0], MODEL[1])
print 'INIT_NET = ', INIT_NET
PREDICT_NET = os.path.join(CAFFE_MODELS, MODEL[0], MODEL[2])
print 'PREDICT_NET = ', PREDICT_NET
if not os.path.exists(INIT_NET):
print INIT_NET, ' not found!'
else:
print 'found ', INIT_NET, '...now looking for ', PREDICT_NET
if not os.path.exists(PREDICT_NET):
print "Caffe model file, ", PREDICT_NET, 'was not found!'
else:
print 'all needed files found! loading the model in the next block'
##load and transform image
batches = []
nBatches = int(math.ceil(1000.0/BATCH_SIZE))
i = 0
print 'Number of batches: ', nBatches
print 'Pre-processing images...'
##create batches of images
for j in range(0, nBatches):
batch = np.zeros([BATCH_SIZE, 3, 227, 227]).astype(np.float32)
##populate each batch
for k in range(0, BATCH_SIZE):
    if i < len(imgFileList):
        imgFile = imgFileList[i]
        imgTemp = skimage.img_as_float(skimage.io.imread(imgFile)).astype(np.float32)
        imgTemp = rescale(imgTemp, INPUT_IMAGE_SIZE, INPUT_IMAGE_SIZE)
        imgTemp = crop_center(imgTemp, INPUT_IMAGE_SIZE, INPUT_IMAGE_SIZE)
        ##print 'after crop: ', imgTemp.shape

        ##switch to chw
        imgTemp = imgTemp.swapaxes(1, 2).swapaxes(0, 1)
        ##switch to bgr
        imgTemp = imgTemp[(2,1,0), :, :]

        ##remove mean for better results
        imgTemp = imgTemp * 255 - mean

        ##add to batch
        batch[k] = imgTemp
        ##print 'nchw: ', img.shape

    i+=1

batches.append(batch)

print 'Done pre-processing, initializing net...'
gpus = []
##Provides option for multiple GPUs
for i in range(NUM_GPUS):
gpus.append(i)
device_opts = core.DeviceOption(caffe2_pb2.CUDA, 1)
##initialize the neural net
"""
Version 1:
This is the way Caffe2 recommends you perform prediction (with the Predictor interface).
However, by default, as of June 2017, the Predictor interface does not
utilize GPU computation. So, it is much slower to run prediction in this way.
"""
##with open(INIT_NET, 'rb') as f:
##init_net = f.read()
##with open(PREDICT_NET, 'rb') as f:
##predict_net = f.read()
##p = workspace.Predictor(init_net, predict_net)
##print 'Running net...'
##tic = time.time()
##for batch in batches:
##p.run([batch])
##toc = time.time() - tic
"""
Version 2:
This is another way to perform prediction that does utilize the GPU (the GPU usage
is specified by the previously defined device_opts). The network initilaization takes
much longer, but the net itself runs much faster.
"""
init_def = caffe2_pb2.NetDef()
with open(INIT_NET, 'rb') as f:
init_def.ParseFromString(f.read())
init_def.device_option.CopyFrom(device_opts)
workspace.RunNetOnce(init_def.SerializeToString())
net_def = caffe2_pb2.NetDef()
with open(PREDICT_NET, 'rb') as f:
net_def.ParseFromString(f.read())
net_def.device_option.CopyFrom(device_opts)
workspace.CreateNet(net_def.SerializeToString())
print 'Running net...'
##timing
tic = time.time()
#`#run the net and return prediction
for batch in batches:
workspace.FeedBlob('data', batch, device_option=device_opts)
workspace.RunNet('AlexNet', 1)
toc = time.time() - tic
##results = workspace.FetchBlob('prob')
print 'Time elapsed running net: ', toc, ' seconds'
print 'Images per second: ', 1000.0/toc
Error1 :
BATCH SIZE:  1
Config set!
Functions set.
Traceback (most recent call last):
File "alexnet_pretrain.py", line 87, in
mean = np.load(MEAN_FILE).mean(1).mean(1)
File "/usr/lib/python2.7/dist-packages/numpy/lib/npyio.py", line 401, in load
"Failed to interpret file %s as a pickle" % repr(file))
IOError: Failed to interpret file '/local/caffe2/build/caffe2/python/models/bvlc_alexnet/ilsvrc_2012_mean.npy' as a pickle
Error2:
BATCH SIZE:  1
Config set!
Functions set.
Mean was set to:  128
INIT_NET =  /local/caffe2/build/caffe2/python/models/bvlc_alexnet/exec_net.pb
PREDICT_NET =  /local/caffe2/build/caffe2/python/models/bvlc_alexnet/predict_net.pb
found  /local/caffe2/build/caffe2/python/models/bvlc_alexnet/exec_net.pb ...now looking for  /local/caffe2/build/caffe2/python/models/bvlc_alexnet/predict_net.pb
all needed files found! loading the model in the next block
Number of batches:  1000
Pre-processing images...
Done pre-processing, initializing net...
Traceback (most recent call last):
File "alexnet_pretrain.py", line 193, in
init_def.ParseFromString(f.read())
File "/usr/lib/python2.7/dist-packages/google/protobuf/message.py", line 186, in ParseFromString
self.MergeFromString(serialized)
File "/usr/lib/python2.7/dist-packages/google/protobuf/internal/python_message.py", line 841, in MergeFromString
if self._InternalParse(serialized, 0, length) != length:
File "/usr/lib/python2.7/dist-packages/google/protobuf/internal/python_message.py", line 866, in InternalParse
new_pos = local_SkipField(buffer, new_pos, end, tag_bytes)
File "/usr/lib/python2.7/dist-packages/google/protobuf/internal/decoder.py", line 827, in SkipField
return WIRETYPE_TO_SKIPPER[wire_type](buffer, pos, end)
File "/usr/lib/python2.7/dist-packages/google/protobuf/internal/decoder.py", line 797, in _RaiseInvalidWireType
raise _DecodeError('Tag had invalid wire type.')
Got the AlexNet model from here https://github.com/caffe2/models/tree/master/bvlc_alexnet
Any help is appreciated, Thanks

For error1,I think you can remove ilsvrc_2012_mean.npy from the models folder.


Cloudfront storage documentation issue

So, after restart the registry, it stopped working . Then I got the error bellow in my logs.
goroutine 1 [running]:
github.com/docker/distribution/registry/storage/driver/middleware/cloudfront.newCloudFrontStorageMiddleware(0xffd980, 0xc0004e0450, 0xc000427020, 0xa, 0xc>
        /go/src/github.com/docker/distribution/registry/storage/driver/middleware/cloudfront/middleware.go:141 +0x128f
github.com/docker/distribution/registry/storage/driver/middleware.Get(0xc0000372c0, 0xa, 0xc000427020, 0xffd980, 0xc0004e0450, 0xd18720, 0xfe14f0, 0x0, 0x>
        /go/src/github.com/docker/distribution/registry/storage/driver/middleware/storagemiddleware.go:34 +0x14b
github.com/docker/distribution/registry/handlers.applyStorageMiddleware(0xffd980, 0xc0004e0450, 0xc000466520, 0x1, 0x1, 0xc000360be0, 0xc0001d3b08, 0x0, 0>
        /go/src/github.com/docker/distribution/registry/handlers/app.go:989 +0xc9
github.com/docker/distribution/registry/handlers.NewApp(0xff1820, 0xc0004270b0, 0xc000456000, 0x0)
        /go/src/github.com/docker/distribution/registry/handlers/app.go:153 +0x800
github.com/docker/distribution/registry.NewRegistry(0xff1820, 0xc0004270b0, 0xc000456000, 0xc000456000, 0x0, 0x0)
        /go/src/github.com/docker/distribution/registry/registry.go:105 +0x1dc
github.com/docker/distribution/registry.glob..func1(0x171ae40, 0xc00003bb80, 0x1, 0x1)
        /go/src/github.com/docker/distribution/registry/registry.go:63 +0x187
github.com/docker/distribution/vendor/github.com/spf13/cobra.(*Command).execute(0x171ae40, 0xc00003bb40, 0x1, 0x1, 0x171ae40, 0xc00003bb40)
        /go/src/github.com/docker/distribution/vendor/github.com/spf13/cobra/command.go:495 +0x191
github.com/docker/distribution/vendor/github.com/spf13/cobra.(*Command).Execute(0x171afe0, 0xc0001d3f88, 0xc00007a058)
        /go/src/github.com/docker/distribution/vendor/github.com/spf13/cobra/command.go:560 +0x2f4
main.main()
        /go/src/github.com/docker/distribution/cmd/registry/main.go:23 +0x2d

Then I went to the documentation at: https://docs.docker.com/registry/configuration/#cloudfront. I checked my config.yml and I dind't found out
any missconfiguration as you can see bellow.
log:
  fields:
    service: my-registry
http:
  addr: :5000
  host: https://docker-registry.****.com
storage:
  s3:
    accesskey: *******
    secretkey: *******
    region: us-east-1
    bucket: ****-s3-bucket
    encrypt: true
    secure: true
    v4auth: true
    chunksize: 5242880
    rootdirectory: /
  redirect:
    disable: true
middleware:
  storage:
    - name: cloudfront
      options:
        baseurl: https://*******.cloudfront.net/
        privatekey: /etc/docker/cloudfront/pk-******.pem
        keypairid: *******

As you can see, all required storage options are in there.
baseurl
privatekey
keypairid
After that I went to check the code and then a reliazed that there is a undocumented storage options "ipfilteredby" that is checked in the line:
https://github.com/docker/distribution/blob/b75069ef13a1de846c0cdf964f5917f5b00c1a47/registry/storage/driver/middleware/cloudfront/middleware.go#L141
So, I added the ipfilteredby to the options and then the registry works again.
....
middleware:
  storage:
    - name: cloudfront
      options:
        baseurl: https://*******.cloudfront.net/
        privatekey: /etc/docker/cloudfront/pk-******.pem
        keypairid: *******
        ipfilteredby: ""

I think you can update the documentation with that option as required, set a default value or not check the type (string) if it's not a required option.
https://docs.docker.com/registry/configuration/#cloudfront
https://github.com/docker/distribution/blob/b75069ef13a1de846c0cdf964f5917f5b00c1a47/registry/storage/driver/middleware/cloudfront/middleware.go#L141
# docker version
Client:
 Version:           18.06.1-ce
 API version:       1.38
 Go version:        go1.10.6
 Git commit:        e68fc7a
 Built:             Tue Aug 21 17:16:31 2018
 OS/Arch:           linux/amd64
 Experimental:      false

Server:
 Engine:
  Version:          18.06.1-ce
  API version:      1.38 (minimum version 1.12)
  Go version:       go1.10.6
  Git commit:       e68fc7a
  Built:            Tue Aug 21 17:16:31 2018
  OS/Arch:          linux/amd64
  Experimental:     false

Containers: 2
 Running: 2
 Paused: 0
 Stopped: 0
Images: 21
Server Version: 18.06.1-ce
Storage Driver: overlay
 Backing Filesystem: extfs
 Supports d_type: true
Logging Driver: json-file
Cgroup Driver: systemd
Plugins:
 Volume: local
 Network: bridge host macvlan null overlay
 Log: awslogs fluentd gcplogs gelf journald json-file logentries splunk syslog
Swarm: inactive
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: 468a545b9edcd5932818eb9de8e72413e616e86e
runc version: 69663f0bd4b60df09991c08812a60108003fa340
init version: v0.13.2 (expected: fec3683b971d9c3ef73f284f176672c44b448662)
Security Options:
 seccomp
  Profile: default
 selinux
Kernel Version: 4.14.96-coreos
Operating System: Container Linux by CoreOS 1967.4.0 (Rhyolite)
OSType: linux
Architecture: x86_64
CPUs: 2
Total Memory: 3.853GiB
Name: ip-10-1-14-30.ec2.internal
ID: NNS5:WQJG:3WGS:XMME:XQBE:UEN3:CSIM:X52U:K7UJ:HPNU:X6EC:CECC
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): true
 File Descriptors: 40
 Goroutines: 70
 System Time: 2019-01-31T15:30:06.187472756Z
 EventsListeners: 1
Registry: https://index.docker.io/v1/
Labels:
Experimental: false
Insecure Registries:
 <IP_ADDRESS>/8
Live Restore Enabled: false

Thanks for the detailed bug report. The latest docs are there in docs/ directory which has documented ipfilteredby. The documentation at docs.docker.com is outdated and I am not sure when it is updated. Moreover, the doc says that it is optional but is still required which is another bug getting fixed in #2837. Since the doc is up to date in https://github.com/docker/distribution/blob/master/docs/configuration.md, lmk if you are ok if I closed this issue.

Hello @manishtomar
You can close this issue. Thanks for the feedback.

Closing as fixed in #3088 and generally, outdated. Feel free to reopen.


Refactored cephcluster_tests.go

Instead of explicitly writing each test case within a function, it is reorganized to follow a table driven approach. The refactoring is done for cephClusterCreate, cephClusterUpdate,cephClusterWithNoConditions, cephClusterWithNegativeConditions
Signed off by: Priyanka Jiandani<EMAIL_ADDRESS>

@jarrpa Please review

/retest


Error while using Drift on embeddings

Hi, I'm redoing the notebook on encoders with Drift which was previously univariate detection.
I'm facing the below error when running the embedding drift methods.

cc @elenasamuylova

Hey @SangamSwadiK ,
Thank you for letting us know!
Could you please share with us a link to your colab so that we could see the whole error trace together with the code?

Hi @SangamSwadiK !
Thanks for letting us know about this issue! We have fixed the error and the changes will be available in the next release.
By the way, we noticed that you're using the "p_value" parameter in the drift detection methods in your notebook. Just a heads up, we've renamed it to "quantile_probability" in the new release, so you'll need to make that change in your notebook.
p_value=0.05 -> quantile_probability=0.95
If you have any inquiries, please do not hesitate to ask us!

fixed in https://github.com/evidentlyai/evidently/pull/586


Not able to link in a new tab/window

HI:
I would like to use something like target="_blank" at the HTML a component, but in the Link component of react-router:
Basically, link to my react component opening it in a new tab or browser window. Is this possible?
Thanks

+1

See #1510


Machines should not carry redstone signal

It seams that machines can carry redstone signals just like normal blocks, this means you can't place the machines right next to eachother without them shareing the redstone signal
 if the bottom machine gets a redstone signal it's transfered to the machine above.

There is a solution for this, disable the "Strong Signal" check box on the redstone conduits.

You are absolutly right, my fault, delete this issue please.
Sorry.


Add migration link changelog

Description
I added a link to the migration guides at the top of the changelog.
Since the website did not have an index for the migration guides, I added it.
And I also used that opportunity to fix the changelog tests which were broken due to username changes.
Fixes #3213
Checklist

[X] I have reviewed my own code
[X] I have added tests (Technically I fixed some tests)

Can you run prettier to make the CI green?
cd website
npm run fmt:write

Done!


fix: don't honor THOR's recommended_min_amount_in

Description
Precisely what the comment says - recommended min. amounts are way too high at the moment.
Pull Request Type

[x] :bug: Bug fix (Non-breaking Change: Fixes an issue)
[ ] :hammer_and_wrench: Chore (Non-breaking Change: Doc updates, pkg upgrades, typos, etc..)
[ ] :nail_care: New Feature (Breaking/Non-breaking Change)

Issue (if applicable)

closes https://github.com/shapeshift/web/issues/5621
Risk

Theoretically none - if we get a quote, it doesn't matter whether or not we handle this minimum or not. It's just the THOR-recommended minimum to get an optimal-enough trade, but effectively blocks trades of less than a few hundred bucks.
Testing

Smallish THOR trades amounts are happy
Expected amount matches the THORSwap ones

Engineering

Operations

Screenshots (if applicable)

Amount actually too low

Happy THOR quote - don't honor recommended minimum and display the quote

Happy THOR trade

Current dependencies on/for this PR:

develop

PR #5624   👈

This stack of pull requests is managed by Graphite.

Discussed this one out - agreed to close it instead of adding some UI warning since the minimum is the state of affairs for the network atm - trying to do anything lesser than that will result in refunds at best, failed refunds worst cast scenario.


List available steps in project

Is your feature request related to a problem? Please describe.
When writing test specifications, it can be difficult to know if I have already implemented a particular step, especially when my step implementations are divided up into several files.
Describe the solution you'd like
It would be very helpful to have a complete list of the steps I have implemented in a project so I don't end up re-implementing a step I already have. This list could be accessed from IDE plugins, and through the Gauge CLI.
For example, gauge list --steps specs/ would find all the steps referenced in the specs folder, confirm they are implemented, and return a list to the user.
Describe alternatives you've considered
Currently I can use autocompletion in the VS code plugin to check if a similar step has been implemented, but this requires me to guess the first few characters of the step description.

Thanks for the idea @CDFriend. The reason we did not include this in the gauge list command was because the number of steps can be quite high.
A while ago I was wondering if there can be a mechanism to detect duplicate steps (either using the step text or the implementation). Although, I haven't investigated it much.

Thanks for the idea @CDFriend. The reason we did not include this in the gauge list command was because the number of steps can be quite high.
A while ago I was wondering if there can be a mechanism to detect duplicate steps (either using the step text or the implementation). Although, I haven't investigated it much.

Understandable! That'd be a lot of information to come out of one command. It could definitely still be useful if passed into grep or a similar tool though - that way a user could quickly search through steps on the command line.
Ultimately what I'd love to see is a feature in the VS Code plugin where we can see all steps in a list in the UI and filter through them by name. Initially I thought the "get all steps" functionality wasn't available in Gauge core but it looks like we can access that information through the gRPC API. Maybe this is a feature that should be requested in the gauge-vscode repository.
  /// [GetAllStepsRequest](#gauge.messages.GetAllStepsRequest)
  GetAllStepsRequest allStepsRequest = 7;
  /// [GetAllStepsResponse](#gauge.messages.GetAllStepsResponse)
  GetAllStepsResponse allStepsResponse = 8;

Understandable! That'd be a lot of information to come out of one command. It could definitely still be useful if passed into grep or a similar tool though - that way a user could quickly search through steps on the command line.
Ultimately what I'd love to see is a feature in the VS Code plugin where we can see all steps in a list in the UI and filter through them by name. Initially I thought the "get all steps" functionality wasn't available in Gauge core but it looks like we can access that information through the gRPC API. Maybe this is a feature that should be requested in the gauge-vscode repository.
  /// [GetAllStepsRequest](#gauge.messages.GetAllStepsRequest)
  GetAllStepsRequest allStepsRequest = 7;
  /// [GetAllStepsResponse](#gauge.messages.GetAllStepsResponse)
  GetAllStepsResponse allStepsResponse = 8;

You're right. GetAllStepsRequest is available via the gauge API. However gauge-vscode does not use the grpc API, it uses the Language Server Protocol which in turn uses json-rpc under the hood.
That said, there's no reason why this cannot be added. The challenge is to get a good UX. Where would you like to see this information?

You're right. GetAllStepsRequest is available via the gauge API. However gauge-vscode does not use the grpc API, it uses the Language Server Protocol which in turn uses json-rpc under the hood.
That said, there's no reason why this cannot be added. The challenge is to get a good UX. Where would you like to see this information?

If you'll excuse my MS paint mockup...

In essence, we were thinking about having a separate view similar to the Gauge test explorer where you could search through steps. Clicking on any given step would take you to the implementation.
My colleague also suggested grouping steps together in a tree by their alias. For example:
Given I am logged in
> And I am logged in
> Log in

If you'll excuse my MS paint mockup...

In essence, we were thinking about having a separate view similar to the Gauge test explorer where you could search through steps. Clicking on any given step would take you to the implementation.
My colleague also suggested grouping steps together in a tree by their alias. For example:
Given I am logged in
> And I am logged in
> Log in

Let me see if there's a way to wire up such a view with vscode. Will then need to figure out who is responsible for the search/filter (vscode or gauge language server).
On a different note, would you be interested in sending a contribution (I am happy to help in any way I can)?

Let me see if there's a way to wire up such a view with vscode. Will then need to figure out who is responsible for the search/filter (vscode or gauge language server).
On a different note, would you be interested in sending a contribution (I am happy to help in any way I can)?

Absolutely - I'm happy to contribute some code for this feature. Just hoping to establish a design before getting too far into building things.
I can start digging into the search/filter part of this in the near future.

Absolutely - I'm happy to contribute some code for this feature. Just hoping to establish a design before getting too far into building things.
I can start digging into the search/filter part of this in the near future.

What are.your thoughts on VSCode's Symbol Search - https://code.visualstudio.com/docs/editor/editingevolved#_open-symbol-by-name
If we follow VSCode's UX, Steps could be symbols that can be searched and navigated to via Symbol Search.
Implementationwise this would mean hooking up a WorkspaceSymbolProvider

What are.your thoughts on VSCode's Symbol Search - https://code.visualstudio.com/docs/editor/editingevolved#_open-symbol-by-name
If we follow VSCode's UX, Steps could be symbols that can be searched and navigated to via Symbol Search.
Implementationwise this would mean hooking up a WorkspaceSymbolProvider

Sounds like a great idea. From what I can see in VS code, we can currently search through Gauge scenarios in VS code, but not Gauge steps.

Sounds like a great idea. From what I can see in VS code, we can currently search through Gauge scenarios in VS code, but not Gauge steps.

I had some time, so tried to look into this a bit further.
So, we have

a way to retrieve all the step texts (without the location) from the implementations,
a way to retrieve location of a step, given it's text
we do not have a way to retrieve all implemented steps with their location

The location becomes relevant because in the symbol search box, the action is to navigate to the symbol (i.e. search + select + Enter).
My first gut was to pick the steps from the spec files, but that would be incorrect for two reasons:

a step can be used multiple times, hence there'll be a lot of duplicates
it will not include unused steps


Need help in filtering the unwanted fields in elasticsearch.

Please use the following questions as a guideline to help me answer
your issue/question without further inquiry. Thank you.
Which version of Elastic are you using?
[ ] elastic.v6 (for Elasticsearch 6.x)
Please describe the expected behavior.
As a part of my elasticsearch Im getting entire structure as json.
type SearchHit struct {
Score          *float64                       json:"_score"          // computed score
Index          string                         json:"_index"          // index name
Type           string                         json:"_type"           // type meta field
Id             string                         json:"_id"             // external or internal
Uid            string                         json:"_uid"            // uid meta field (see MapperService.java for all meta fields)
Routing        string                         json:"_routing"        // routing meta field
Parent         string                         json:"_parent"         // parent meta field
Version        *int64                         json:"_version"        // version number, when Version is set to true in SearchService
Sort           []interface{}                  json:"sort"            // sort information
Highlight      SearchHitHighlight             json:"highlight"       // highlighter information
Source         *json.RawMessage               json:"_source"         // stored document source
Fields         map[string]interface{}         json:"fields"          // returned (stored) fields
Explanation    *SearchExplanation             json:"_explanation"    // explains how the score was computed
MatchedQueries []string                       json:"matched_queries" // matched queries
InnerHits      map[string]*SearchHitInnerHits json:"inner_hits"      // inner hits with ES >= 1.5.0
Nested         *NestedHit                     json:"_nested"         // for nested inner hits
}
I need only following in Json
Index          string                         json:"_index"          // index name
Id             string                         json:"_id"             // external or internal
Source         *json.RawMessage               json:"_source"         // stored document source
How to get only these three fields in Json results and remove the other unwanted fields ?

I'm performing the search like given below.
fetchContext := elastic.NewFetchSourceContext(true).Include("date")
    res, err := client.Search().
            Index("xxxx").
            Query(query).
            FilterPath(
                    "hits.total",
                    "hits.hits._id",
                    "hits.hits._source",
                    "hits.hits._index",
            ).
            FetchSourceContext(fetchContext).
            Pretty(true).
            From(0).Size(1).
            Do(ctx)

Then
hit, err := json.Marshal(res.Hits.Hits)
if err != nil {
            fmt.Printf("Error:%s\n", err)
    } else {
            fmt.Println(string(hit))
    }

this prints outcome as
[{"_score":null,"_index":"xxxx","_type":"","_id":"yyyy","_uid":"","_routing":"","_parent":"","_version":null,"sort":null,"highlight":null,"_source":{"date":1.529907000446799E9},"fields":null,"_explanation":null,"matched_queries":null,"inner_hits":null,"_nested":null}]
But I need only :
_index
_id
_source

I could add ,omitempty to the optional parts. But that'd only solve the problem of not printing it to stdout. As you typically don't care about the printed output (or the serialized JSON coming out of ES), I'd consider that a minor issue. Just don't use the properties you don't need. What I really care about is what's happening on the wire. You can only see that when enabling a network sniffer or the trace log in Elastic (see the Wiki).

See #927

As you said I can just ignore the properties that I don't need. But problem is, when I marshal the data, buffer size will increase due to unwanted fields. I was able to get the data that I need by altering the structure like given below and reading it.
I have added the omitempty field as you suggested.
type SearchHit struct {
Score          *float64                       json:"_score,omitempty"          // computed score
Index          string                         json:"_index,omitempty"          // index name
Type           string                         json:"_type,omitempty"           // type meta field
Id             string                         json:"_id,omitempty"             // external or internal
Uid            string                         json:"_uid,omitempty"            // uid meta field (see MapperService.java for all meta fields)
Routing        string                         json:"_routing,omitempty"        // routing meta field
Parent         string                         json:"_parent,omitempty"         // parent meta field
Version        *int64                         json:"_version,omitempty"        // version number, when Version is set to true in SearchService
Sort           []interface{}                  json:"sort,omitempty"            // sort information
Highlight      SearchHitHighlight             json:"highlight,omitempty"       // highlighter information
Source         *json.RawMessage               json:"_source,omitempty"         // stored document source
Fields         map[string]interface{}         json:"fields,omitempty"          // returned (stored) fields
Explanation    *SearchExplanation             json:"_explanation,omitempty"    // explains how the score was computed
MatchedQueries []string                       json:"matched_queries,omitempty" // matched queries
InnerHits      map[string]*SearchHitInnerHits json:"inner_hits,omitempty"      // inner hits with ES >= 1.5.0
Nested         *NestedHit                     json:"_nested,omitempty"
    // Shard
    // HighlightFields
    // SortValues
    // MatchedFilters

}
Since, this is optional field can you push this changes to official code (version 6.0 ) if possible ?
https://github.com/olivere/elastic/issues/927
If not ,is there a way to checkout stable version 7.0( if available which has this fix)

Buffer size will not increase as the struct still has those fields, regardless of omitempty or not. The fields will still be there, with their default values.
Either I don't understand your use case or you have a misunderstanding between *SearchHit being an in-memory data structure and the JSON output of the struct when you serialize it.
Can you tell me why you need the generated JSON output from the *SearchHit?

Okay, not quite sure why you'd wanna do that. But anyway, I will add the omitempty field tag in 6.2.9. Notice that I won't do that for all request/response structs of Elastic, as it's not the primary purpose of Elastic to serve as a proxy. There are better solutions for that.

https://github.com/olivere/elastic/milestone/18

Hi ...May I know how to checkout the code with these changes ?
I was unable to find the code in the below URL
https://github.com/olivere/elastic/milestone/18
that you have shared .

Thanks for the update. Will wait for 6.2.9 over the weekend.


Encryption mechanism

The original proof of concept uses the built-in password-based encryption provided by iTunes. An open question is whether this should be supported long-term. Some concerns:

Relies on password strength. This could be an issue, especially if users choose to store their backup in the public IPFS network
There are some manifest/metadata files that don't seem to be encrypted. How sensitive is this data?

Possible ideas:

Custom encryption mechanism that requires a more strict key-management integration and encrypts manifests
Use the built-in encryption, but enforce a password strength
Leave it up to the user

More info on the metadata:
https://www.theiphonewiki.com/wiki/ITunes_Backup
There is definitely sensitive data.


Reuse  Publisher instance

Hi,
I have a question about reusing Publisher object in publishing example:
Publisher publisher = null;
try {
  publisher = Publisher.newBuilder(topic).build();
  ByteString data = ByteString.copyFromUtf8("my-message");
  PubsubMessage pubsubMessage = PubsubMessage.newBuilder().setData(data).build();
  ApiFuture<String> messageIdFuture = publisher.publish(pubsubMessage);
  ApiFutures.addCallback(messageIdFuture, new ApiFutureCallback<String>() {
    public void onSuccess(String messageId) {
      System.out.println("published with message id: " + messageId);
    }

    public void onFailure(Throwable t) {
      System.out.println("failed to publish: " + t);
    }
  }, MoreExecutors.directExecutor());
  //...
} finally {
  if (publisher != null) {
    publisher.shutdown();
    publisher.awaitTermination(1, TimeUnit.MINUTES);
  }
}

When multiple events (thousands, milions) are sent, then each time a new Publisher is created and takes a Thread. Whats a general approach to asynchronously send a big amount of events?
What do you think to keep Publisher instance in cache/static field and do not shutdown? Whats a optimum time for awaitTermination

same question

I opened a similar issue a while back: https://github.com/googleapis/java-pubsub/issues/86

That is correct. We have opened an issue in our public issue tracker to add support to our Publisher client to allow users to publish to multiple topics.


[5.8] Simplify encrypter hmac verification

I don't see the reason why the hmac needs to be doubly-hashed. Two fewer hashing calls, one fewer random call.

I'm not changing the encrypter without good reason. I've already had it reviewed by people smarter than me at encryption.

@taylorotwell Can you clarify why the double HMAC calculation is needed?
This is the relevant commit:
https://github.com/laravel/framework/commit/c3e6e84229f790077b9a2b43976d326f6ef12b24

Likely https://crypto.stackexchange.com/questions/30084/double-randomised-hmac-verification-to-prevent-timing-attack

@mfn
I see that's pretty smart. But it is redundant since the now used hash_equals and the previously used StringUtils::equals already prevent timing attacks. I suggest removing the superfluous HMAC calculation to keep the code lean and easier to understand. If not that then it would help to have a comment in the code that explains the reasoning.


Manually publish to ReadTheDocs

Connect the GitHub repo to ReadTheDocs and manually publish at least once.

ReadTheDocs page registered and created:
https://readthedocs.org/projects/visumorph/
I will close this issue once we confirm the rendered website is OK.

I confirm it is ok!

I can see it. But I guess the content is missing

Should be resolved in the latest merge (#65)
This latest build passed (https://readthedocs.org/projects/visumorph/builds/23260082/)
@Marcony1  Can you help to check if the site is rendered properly?

@SoloSynth1 LGTM, Thank you.

Thanks! Closing this issue.


Rfc/governance/patches

Hi @didier-wenzek
here are my suggestions for https://github.com/thin-edge/thin-edge.io/pull/1203 for some of the sections below the "Repository maintainance" header.
Feel free to suggest changes or clearification at any point - I'm happy to discuss! :grinning:

I agree with all the points. However, I'm really busy today and will take the time tomorrow for a proper review.

I okay to merge this PR with minor exceptions. Since we agreed on these, I will merge and fix them (unless you have time to do it sooner ;-)).


update debian from 10 to 11

Currently we have error ./altv-voice-server: /usr/lib/x86_64-linux-gnu/libstdc++.so.6: version GLIBCXX_3.4.26 not found (required by ./altv-voice-server)
Altv says that we need at least Debian 11 to use all those stuff

seems reasonable.


Docker Toolbox for Windows, Swarm mode, services connection refused

Docker toolbox for windows
Services are refusing connection when using swarm mode on Docker toolbox for Windows
Note: (docker toolbox) Docker is running in a linux VM inside my local machine with an ip address of <IP_ADDRESS>
Steps to Reproduce the issue -
Step 1: Initialize the swarm mode
$ docker swarm init --advertise-addr <IP_ADDRESS>
Swarm initialized: current node (yfgi1sjocv9jq74e97g6ya9fm) is now a manager.
Step 2: pull an nginx image
$ docker pull nginx
Using default tag: latest
latest: Pulling from library/nginx
a5a6f2f73cd8: Pull complete
1ba02017c4b2: Pull complete
33b176c904de: Pull complete
Digest: sha256:5d32f60db294b5deb55d078cd4feb410ad88e6fe77500c87d3970eca97f54dba
Status: Downloaded newer image for nginx:latest
Step 3: create nginx service and expose it to port 7070
$ docker service create --name svcnginx -p 7070:80 nginx
1rmtfr7hs0oashfmg6ty57nox
overall progress: 1 out of 1 tasks
1/1: running
verify: Service converged
Step 4: Check the service we just created
$ docker service ls
ID                  NAME                MODE                REPLICAS
IMAGE               PORTS
1rmtfr7hs0oa        svcnginx            replicated          1/1
nginx:latest        *:7070->80/tcp
Step 5: Check the containers
$ docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED
STATUS              PORTS               NAMES
1d681e3d0ea7        nginx:latest        "nginx -g 'daemon of"   16 seconds ago
Up 15 seconds       80/tcp              svcnginx.1.58wzzqwbrcd3vmt9ymyxy2mmn
Step 6: Try to access the running nginx service
$ curl $(docker-machine ip default):7070
curl: (7) Failed to connect to <IP_ADDRESS> port 7070: Connection refused
Expected O/P:
$ curl <IP_ADDRESS>:7070

Welcome to nginx!

Welcome to nginx!
If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.
For online documentation and support please refer to
nginx.org.
Commercial support is available at
nginx.com.
Thank you for using nginx.

I am new to docker not really sure what i am doing wrong. Practiced these same steps here
"https://www.katacoda.com/courses/docker/playground" and everything work fine.
Running standalone nginx contianer also works fine without creating service
$ docker run -d -p 7000:80 nginx
$ curl $(docker-machine ip default):7000
% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
Dload  Upload   Total   Spent    Left  Speed
100   612  100   612    0     0  40800      0 --:--:-- --:--:-- --:--:-- 40800

Welcome to nginx!

Welcome to nginx!
If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.
For online documentation and support please refer to
nginx.org.
Commercial support is available at
nginx.com.
Thank you for using nginx.

Update: this works on a public network but fails on a protected network. I am still trying to understand what is wrong with the machines on protected networks which prevents docker swarm mode services to work.

Update I found this-
Docker Swarm's routing mesh feature is not yet supported on Windows, but users seeking to publish ports for their service endpoints can do so today using publish-port mode.
https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/swarm-mode#publish-ports-for-service-endpoints

The problem was proxy settings on the boot2docker vm i was running.


I Can not run the Demo

I have download cef3.3359_window64
Copy all files from /Release to the demo folder.
Copy all files from /Resources to demo folder.
3.when I run cefglue_winapi_demo.exe , it is a blank

Copy all files from /Release to the demo folder.
Copy all files from /Resources to demo folder.
When I run  cefglue_winapi_demo.exe

Please see the guide #7
"Get Cef binaries from download page. Base Chromium version 59 or above (64-bit) with version matching Chromely.Unofficial.CefGlue.NetStd installed."
Current version of Chromely.Unofficial.CefGlue.NetStd is 64.0.3282.119.
So you will need cef_binary_3.3282.1741.gcd94615_windows64_minimal.
Please let me know if you need more info.

Also the log would have more info.

I have download cef_binary_3.3282.1741.gcd94615_windows64_minimal.
and Copy all files from /Release to the demo folder. Copy all files from /Resources to demo folder.
debug.log
[0512/005240.311:WARNING:resource_bundle.cc(368)] locale_file_path.empty() for locale
[0512/005240.311:ERROR:main_delegate.cc(683)] Could not load locale pak for en-US
[0512/005240.311:ERROR:main_delegate.cc(690)] Could not load cef.pak
[0512/005240.311:ERROR:main_delegate.cc(707)] Could not load cef_100_percent.pak
[0512/005240.311:ERROR:main_delegate.cc(716)] Could not load cef_200_percent.pak
[0512/005240.311:ERROR:main_delegate.cc(726)] Could not load cef_extensions.pak
[0512/005240.311:ERROR:content_client.cc(272)] No data resource available for id 101

I think I see where the confusion might be. The statement should be "copy all files and folders ...".  I will make that change in the wiki. Sorry about that.
It looks like the "locales" folder is missing.
This should be folder layout after the cef binaries merge:

This should be the folder layout when added to the bin folder:

but still
[0512/135201.468:WARNING:resource_bundle.cc(368)] locale_file_path.empty() for locale
[0512/135201.468:ERROR:main_delegate.cc(683)] Could not load locale pak for en-US
[0512/135201.468:ERROR:main_delegate.cc(690)] Could not load cef.pak
[0512/135201.468:ERROR:main_delegate.cc(707)] Could not load cef_100_percent.pak
[0512/135201.468:ERROR:main_delegate.cc(716)] Could not load cef_200_percent.pak
[0512/135201.468:ERROR:main_delegate.cc(726)] Could not load cef_extensions.pak
[0512/135201.484:ERROR:content_client.cc(272)] No data resource available for id 101

I suggest author download it from github，and then run cefglue_winapi_demo.exe
I download cef_binary_3.3282.1741.gcd94615_windows64_minimal. and try it failed

Since you have all the files the application is complaining about, it may be a permission issue on your computer.
But first what Windows version are you running? I am running Windows 10.
Also the "locales" folder should have all the pak files.
I just downloaded the repo and both CefGlue demos ran successfully:

CefGlue_netCore_win10-x64 (add the merged cef binaries and run "cefglue_winapi_netcoredemo.exe")
CefGlueWin64

You can try the self-contained .NET Core build - CefGlue_netCore_win10-x64.
Another alternative is to try the CefSharp project. This will download the required cef binaries when you build the project and references are restored.

Also if you have .NET Core 2.0 installed you can try the cli:
dotnet-cli-Starter-Template

I will close this for now. Please reopen this or start a new thread if you have more questions.


Getter-only properties are no longer automatically ignored

My entity includes a property that only has a getter, e.g. something like this:
public string FullName
{
    get { return FirstName + " " + LastName; }
}

Previously, this was automatically ignored by EF when building the model. But now, I receive the following error:

No backing field could be discovered for property 'Example.User.FullName' and the property does not have a setter. Either use a backing field name that can be matched by convention, annotate the property with a backing field, or define a property setter.

I had to add an explicit entity.Ignore(x => x.FullName) to the model builder, in order to fix this. But before RC1, this was just working fine without having to explcitly ignore properties. When and why was this changed?

This was an intentional change we made. We now include all properties in the model since folks would often not realize that things were being excluded. We should change that error message though to tell you that you can use Ignore(...) or [NotMapped] to exclude it.

Ah, I see, thanks for the information.
I would welcome if there was also a way to disable automatic discovery for entities completely. So that you need to mention all properties explicitly in the model builder in order to get them mapped. Similarly how you can switch from opt-out to opt-in with Json.Net which requires you to annotate all members you actually want to serialize.

@AndriySvyryd is property discovery implemented as a convention? i.e. is it something you would be able to remove once we have a public convention surface?

Yed


expressWinston.logger({ msg }) prints plaintext function instead of return value

I'm having issues with printing out messages when I pass in a function, instead of a string in the options.msg parameter.
Dependencies
Verified this happening on Node v9.2.0 (stable) and Node v8.10.0 (lts).
// package.json dependencies
{
    "express": "^4.16.2",
    "express-winston": "^2.5.0",
    "winston": "^2.4.0"
}

Issue
Given the following configuration:
const winstonLoggerConfig = {
  transports: [
    new winston.transports.Console({
      json: true,
      colorize: false
    })
  ],
  meta: true,

  // passing in a function, instead of a string
  msg: (req, res) => {
    const operationName = (req.body || {}).operationName || "NO-OP";
    return `${res.statusCode} ${req.method} ${req.url} ${operationName} ${res.responseTime}ms`;
  },
  // aside. also verified that function() {} fails as well

  requestWhitelist: [...expressWinston.requestWhitelist, "body"],
  expressFormat: false,
  colorize: false
};

// ...
app.use(expressWinston.logger(winstonLoggerConfig));

Expected Output
I expected to see "message": "200 POST /graphql SignInMutation 338ms":
{
  "res": {
    "statusCode": 200
  },
  "req": {
    "url": "/graphql",
    "headers": {
      "host": "localhost:3000",
      "connection": "keep-alive",
      "content-length": "395",
      "accept": "*/*",
      "origin": "http://localhost:3001",
      "authorization": "null",
      "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
      "content-type": "application/json",
      "dnt": "1",
      "referer": "http://localhost:3001/sign-in",
      "accept-encoding": "gzip, deflate, br",
      "accept-language": "en-CA,en-GB;q=0.9,en-US;q=0.8,en;q=0.7"
    },
    "method": "POST",
    "httpVersion": "1.1",
    "originalUrl": "/graphql",
    "query": {},
    "body": {
      "operationName": "SignInMutation",
      "variables": {
        "email"<EMAIL_ADDRESS>        "password": "supersecret"
      },
      "query": "mutation SignInMutation($email: String!, $password: String!) {\n  signinUser(email: {email: $email, password: $password}) {\n    token\n    user {\n      _id\n      username\n      createdAt\n      updatedAt\n      email\n      emailVerified\n      __typename\n    }\n    __typename\n  }\n}\n"
    }
  },
  "responseTime": 338,
  "level": "info",
  "message": "200 POST /graphql SignInMutation 338ms"
}

Actual Output
Instead I see "message": "(req, res) => {\n    const operationName = (req.body || {}).operationName || \"NO-OP\";\n    return `${res.statusCode} ${req.method} ${req.url} ${operationName} ${res.responseTime}ms`;\n  }" :
{
  "res": {
    "statusCode": 200
  },
  "req": {
    "url": "/graphql",
    "headers": {
      "host": "localhost:3000",
      "connection": "keep-alive",
      "content-length": "395",
      "accept": "*/*",
      "origin": "http://localhost:3001",
      "authorization": "null",
      "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
      "content-type": "application/json",
      "dnt": "1",
      "referer": "http://localhost:3001/sign-in",
      "accept-encoding": "gzip, deflate, br",
      "accept-language": "en-CA,en-GB;q=0.9,en-US;q=0.8,en;q=0.7"
    },
    "method": "POST",
    "httpVersion": "1.1",
    "originalUrl": "/graphql",
    "query": {},
    "body": {
      "operationName": "SignInMutation",
      "variables": {
        "email"<EMAIL_ADDRESS>        "password": "supersecret"
      },
      "query": "mutation SignInMutation($email: String!, $password: String!) {\n  signinUser(email: {email: $email, password: $password}) {\n    token\n    user {\n      _id\n      username\n      createdAt\n      updatedAt\n      email\n      emailVerified\n      __typename\n    }\n    __typename\n  }\n}\n"
    }
  },
  "responseTime": 338,
  "level": "info",

  "message": "(req, res) => {\n    const operationName = (req.body || {}).operationName || \"NO-OP\";\n    return `${res.statusCode} ${req.method} ${req.url} ${operationName} ${res.responseTime}ms`;\n  }"
}

Links

Commit hash: https://github.com/udia-software/udia-server/commit/d5dbcd5547f7696d916ceff07134c85502f5acdd
TravisCI build showing logging errors: https://travis-ci.org/udia-software/udia-server/jobs/350503545

https://github.com/bithavoc/express-winston/pull/160

Closing because this is fixed in #160, which has been merged.


Component Name while using react.js

Does this issue occur when all extensions are disabled?: Yes/No

VS Code Version:
OS Version:

Steps to Reproduce:

When the user creates a component in react
now user want to rename the component name
user updates the component name and now it shows error and we cannot use that name of component in the particular project
our projects works well  but the symbol of error show on the vs code which irritates alots while coding.

I Hope You will resolve that issue as soon as possible.
thank you!

Please share some example code which demonstrates the issue


chore(deps): update common-dev-assets digest to a2dbd01

This PR contains the following updates:

Package
Update
Change

common-dev-assets
digest
60783c4 -> a2dbd01

Configuration
📅 Schedule: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined).
🚦 Automerge: Disabled by config. Please merge this manually once you are satisfied.
♻ Rebasing: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox.
👻 Immortal: This PR will be recreated if closed unmerged. Get config help if that's undesired.

[ ] If you want to rebase/retry this PR, click this checkbox.

This PR has been generated by Renovate Bot.

:tada: This PR is included in version 2.1.0 :tada:
The release is available on GitHub release
Your semantic-release bot :package::rocket:


Document the process of using alternative middlewares (Hapi, Koa) instead of Express

Also, would be a good idea to include documentation regarding how to integrate with other middle wares that aren't officially supported.

Are you interested in adding this?

I would love to but I have no idea how to do this in code. Is there any repo or sample anywhere that I can see, then maybe I'll be able to write the docs.

I'm seconding this. I just spent hours trying to figure it out only to realize the whole time I just needed to change routes/middleware to koa in tsoa.json. Took me reading the source files to figure that out. If nothing else, in some obvious place in the documentation (maybe in the getting started guide) I think this should at least be mentioned if nothing else. How to get koa's context object in a request is already mentioned in another page so I'd say might as well. If I have time I'll make a pull request for it.


Unable to include libraries in recent versions of Kha

Steps to replicate:

git clone --recursive https://github.com/KTXSoftware/Empty.git
Edit Empty/khafile.js  to include the line project.addLibrary('$ANY_INSTALLED_LIB); after project declaration, before resolving.
node Kha/make html5 (or any target).

Output:
Creating Kha project.
Error: khafile.js did not call resolve, no project created.
path.js:7
    throw new TypeError('Path must be a string. Received ' + inspect(path));
    ^

TypeError: Path must be a string. Received undefined
    at assertPath (path.js:7:11)
    at Object.join (path.js:1213:7)
    at findLibraryDirectory (/Users/jamiltron/Workspace/Empty/Kha/Tools/khamake/out/Project.js:77:32)
    at Project.addLibrary (/Users/jamiltron/Workspace/Empty/Kha/Tools/khamake/out/Project.js:97:23)
    at eval (eval at <anonymous> (/Users/jamiltron/Workspace/Empty/Kha/Tools/khamake/out/ProjectFile.js:28:17), <anonymous>:6:9)
    at fs.readFile (/Users/jamiltron/Workspace/Empty/Kha/Tools/khamake/out/ProjectFile.js:28:78)
    at tryToString (fs.js:449:3)
    at FSReqWrap.readFileAfterClose [as oncomplete] (fs.js:436:12)

This project builds without the addLibrary line.

Are you on the very very newest version of Kha (and especially khamake)?

Just noticed you're the same guy who asked earlier about ash. If it's still not working please give me more input about your system etc. because it is working for me.

Yes, I just rechecked out Empty to try again and still have this as an issue.
I am running:

OSX 10.11.6
Haxe 3.2.1
Haxelib 3.2.0-rc.3
node 6.2.1

Any other information that's useful?

Did you update the copy of Kha inside of Empty?

D'oh, sorry I forgot to call git submodule foreach --recursive git pull origin master .
Sorry for wasting your time!

Don't worry about it. Update process is too complicated, have to build something into khamake for that.


[firebase_messaging] Android App not registering with firebase

Describe the bug
I am not receiving push notifications sent with Firebase Cloud Messaging. When I create a message from the firebase console the Target step reports: 0% of potential users are eligible for this campaign:0, indicating that the FCM token is not registered with the server. However, if I directly input the token from getToken() into the Send test message dialog the message sends successfully. So I imagine the issue involves the plugin not registering the token.
To Reproduce
Steps to reproduce the behavior:

Followed the installation and Android Integration steps from the README (latest version)
I am also using flutter_local_notifications but I was under the impression that issues with this package were fixed, so I don't think this is the cause of the problem.

hi did you manage to fix this I have the same problem

I left it a few days, and when I came back to it it had started working properly. I have no idea what the issue was, but it might just be some weirdness with the way firebase handles new devices.

Closing in favour of trying the latest messaging version which has since this issue was created been heavily reworked to improve it along with detailed documentation: https://firebase.flutter.dev/docs/messaging/overview
If you still have a problem please raise a new GitHub issue with up to date information and code snippets if possible. Thanks.


Texture mapping appears different

Texture mapping does not match between the exported gltf and Rhino.
Appending the object.GetMeshes mesh array into a single mesh fixes this. I'm doing this for performance but this might be a side effect of something else.
Also, the reason to not join these meshes together I guess is in case they have a different material per face, but it doesn't seem to check if this is the case.

I think we tried per face material support at some point (there was a PR from a user) but it was either buggy or caused a bunch of regressions. Can you attach an example file so I can open a YouTrack issue for the mappings going wrong?


error: can't start new thread

This is under Windows. Runs for a bit and then errors. I've adjusted the number of threads (even down to 1) and still run into it. I'll dig into it a little more and update this issue if I find anything else.

Same issue on Linux Kali

Same on Ubuntu:
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 754, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/home/j4v/Git/aws-inventory/aws_inventory/invoker.py", line 97, in _probe_services
    num_threads=config.MAX_THREADS)
  File "/home/j4v/Git/aws-inventory/aws_inventory/invoker.py", line 188, in thread_work
    worker.start()
  File "/usr/lib/python2.7/threading.py", line 736, in start
    _start_new_thread(self.__bootstrap, ())
error: can't start new thread

Same for me:
**DEBUG:aws_inventory.invoker:[eu-west-1][kinesis] Invoking API "ListStreams". Python name "list_streams".
Exception in thread Thread-1:
Traceback (most recent call last):
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, self.__kwargs)
File "/Users/john_white/code/aws-inventory/aws_inventory/invoker.py", line 97, in _probe_services
num_threads=config.MAX_THREADS)
File "/Users/john_white/code/aws-inventory/aws_inventory/invoker.py", line 188, in thread_work
worker.start()
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 745, in start
_start_new_thread(self.__bootstrap, ())
error: can't start new thread

Use Ubuntu Desktop

@chilithomas I've run it in Ubuntu but the issue is with python.
@bitsandsalsa have you had the time to look into this? For my part it's a blocker.

Ran into this on MacOS Sierra as well

We use a pool of worker threads to call APIs for a given service in a region. The threads were never stopping because they did not know when they were done. The result would be too many threads or too many open connections for the process.

~/gitProjects/aws-inventory/ develop$:python aws_inventory.py --profile MYPROFILE`
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/usr/local/Cellar/python/2.7.14/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/usr/local/Cellar/python/2.7.14/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 754, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/Users/37272/gitProjects/aws-inventory/aws_inventory/invoker.py", line 97, in _probe_services
    num_threads=config.MAX_THREADS)
  File "/Users/37272/gitProjects/aws-inventory/aws_inventory/invoker.py", line 188, in thread_work
    worker.start()
  File "/usr/local/Cellar/python/2.7.14/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 736, in start
    _start_new_thread(self.__bootstrap, ())
error: can't start new thread

From my testing with the develop branch. Let me know if there's any logs I can send over

I'm having trouble reproducing this. In a Windows 10 VM, I could hit it when I get to around 100,000 threads (as reported by threading.active_acount()) if I force a worker pool of around 10,000 threads and wait a few services, but even if somehow your system was creating a thread per API call or even a whole worker pool per API, I don't see how you could hit that many threads. This assumes you are using the default operation blacklist.
Try setting the MAX_THREADS variable in config.py to 1. I am not sure if the bug is caused by something in your AWS account or your execution environment.

Are you sure you were on the develop branch? Try a new clean clone of the develop branch.
Could you run with the --debug commandline option and add some of that output here.
Are you running in some really constrained or minimal environment (e.g., Docker, virtual machine, sandbox)? How much memory is available in the environment? I don't think Homebrew is causing this.
Are you using the default operation blacklist file? Since we retrieve ALL data from APIs that have pagination (not just the first page), I wonder if botocore is spawning a ton of threads to parse all of the pages of responses.
What permissions does your AWS user have? I tried to reproduce the bug using PowerUserAccess policy.

MAX_THREADS: I'd be happy to try that out

Yes I am positive. I can try a clean clone of develop next. However I did not make any changes. That doesn't mean I couldn't have had a PEBCAK
I will try that first and add it in as an edit
I am running directly on MacOS Sierra. I just upgraded to High Sierra since attempting this however I highly doubt that would change anything. I have 16 GB of memory in my laptop
To be honest I just cloned it down. I did not set or change any blacklists. If one comes configured, that is what I am using. If it does not than I am not using one.
That I am not sure. I will do my best to verify if the other tests are unsuccessful

After removing my directory and recloning for specifically the develop branch, my run did complete. I apologize about the incorrect report

Thanks for working through this. @j4v @inquisb @lllama I think the bug may be fixed. If you have a chance, see if this works for you too.

Cleaning out issues. It looks like this was already resolved, so I'm closing this. Thank you for your contribution.


Add flat_map_iter and flatten_iter

The existing flat_map and flatten adaptors expect items that
implement IntoParallelIterator, and they each go through a new round
of parallel task splitting. This often creates smaller task granularity
than people intended, especially if they're doing something simple on
the flattened iterator like collect() or sum().
The new flat_map_iter and flatten_iter expect IntoIterator items,
which are simply flattened directly within the current parallel task.
The iterator doesn't even need to be thread safe, because it never
leaves the thread where it is created, only its items.

Yeah, if we could go back, I think I'd call it par_flat_map for nested parallelism. I suppose we could add that and deprecate flat_map, but then this PR should get a more symmetric name -- maybe ser_flat_map? seq_? We don't have a good convention yet, because serial is usually the unprefixed case.

Hmm, yeah, the idea of deprecating flat_map is appealing, to help steer people towards this new method. I don't love the name seq_flat_map but I do agree that having a symmetric naming convention between par_flat_map and seq_flat_map makes sense.
That said, I feel like seq_flat_map could be confusing, as it could suggest that the entire parallel iterator is somehow made sequential, rather than simply that the items from a single iterator are processed together...hmm.
I wonder if flat_map_par and flat_map_seq is better? (i.e., suffix?) I guess that it's kind of inconsistent with .par_iter instead of .iter.
Another idea: flat_map_par_iter and flat_map_iter?

Oh, I just ignored ser_flat_map, but that's because it's kind of unclear to me what ser stands for -- serialized? serde? :)

Options:

parallel
sequential

par_flat_map
seq_flat_map

par_flat_map
ser_flat_map

flat_map_par_iter
flat_map_iter

I think of these I prefer the first one. I think I favor deprecating the existing methods and recommending the user choose between seq_flat_map and par_flat_map

These should also be compared with flatten:

parallel
sequential

par_flatten
seq_flatten

par_flatten
ser_flatten

flatten_par_iter
flatten_iter

I agree we can strike "ser" -- it sounds like the Game of Thrones honorific here, "Ser Flatten"!
I kind of like the suffix for both flat_map and flatten, as it reads like a verb-object, to flatten an iterator or flatten a parallel iterator.

One other thing I am wondering is whether there is some way to make par_flat_iter more efficient, but that would obviously require some deeper edits. =)

Discussed in gitter today. We decided to keep the PR as is. Our reasoning:

Deprecating the existing method is a bit of a blunt hammer -- we may also eventually be able to make the "to parallel iterator" perform better.
The "verb-object" name makes sense ("flat map an iterator", "flatten an iterator"), and even if you don't consider performance, it may be important for type reasons to have the option to flat-map a non-parallel-iterator.
We retain the option of deprecating and adding an explicit name later.

bors r+


Build Failing

The build is currently failing. Looks like tests aren't passing

@mattnenterprise Sorry for the late response.
I am actually in the process of redoing this package with a cleaner API and better tests: https://github.com/redsync/redsync
I will also look into the this current package soon and try to figure out what is wrong. The odd part is the test passes on my primary machine, but almost nowhere else.

I'm interested in using redsync but am worried about the failing build. Is there any timeline to merge the develop branch? Please let me know if there's anything I can do to help expedite the process.

@mattnenterprise @bfallik It seems the tests were failing due to some change in the tempredis package that was being used to spin up Redis instances for running the tests. I have pushed a fix and tests are now passing: https://drone.io/github.com/hjr265/redsync.go/17
I will still continue working on the new package and eventually deprecate this. But that might take a while as there are a number of suggestions that I am taking into account.

@hjr265 great, thanks!


Let versionCatalogs extension be available in precompiled Kotlin DSL scripts

Fixes https://github.com/gradle/gradle/issues/22468

This PR also splits a large test class into several ones, hence more files changed than only necessary for the fix.
Take a look at the commits to make the review easier.

@bot-gradle test and merge

Your PR is queued. See the queue page for details.

I've triggered a build for you. Click here to see all failures if there's any.


Support home-brew installs for OS X.

Instead of /usr/include/postgresql/libpq-fe.h it should be /usr/local/Cellar/postgresql/9.4.5_2/include/libpq-fe.h for OSX. Is there a way to make the module map file platform dependent?

We're working on it! (:


feat(Data/Nat/Factorial): add descFactorial_le and descFactorial_mul_descFactorial

Add two basic theorems involving falling factorials. Note that descFactorial_mul_descFactorial collapses to Nat.factorial_mul_descFactorial when m = n.

I'm closing this PR and https://github.com/leanprover-community/mathlib4/pull/15665 so that I can reopen then the right way.


Split E2E test as a separate CI jobs

One CI job can only run in searial, with more and more e2e cases added, it will take more time.
And the second reason is that, if one step failed, it will stop.

/assign


Update Universal Sentence Encoder for TF2.0 SavedModel

Hi,
The following models for multi-lingual Universal Sentence Encoder are not available as TF2.0 SavedModel:

https://tfhub.dev/google/universal-sentence-encoder-xling-many/1
https://tfhub.dev/google/universal-sentence-encoder-xling/en-de/1
https://tfhub.dev/google/universal-sentence-encoder-xling/en-es/1
https://tfhub.dev/google/universal-sentence-encoder-xling/en-fr/1

Would it be possible to ask the authors/devs to provide a new version for TF2.0 Saved Model like other multi-lingual USE models: https://tfhub.dev/google/universal-sentence-encoder-multilingual/3
Many thanks

Hi,
I would be highly interested for having https://tfhub.dev/google/universal-sentence-encoder-xling/en-fr/1 as TF2.0 saved model, since it gives more accurate results than the multilangual model.
Thanks !

From conversation with the publisher, TF2 models:
https://tfhub.dev/google/universal-sentence-encoder-cmlm/en-base/1
https://tfhub.dev/google/universal-sentence-encoder-cmlm/multilingual-base/1
should supercede the old TF1 versions (https://tfhub.dev/google/universal-sentence-encoder-xling-many/1,
https://tfhub.dev/google/universal-sentence-encoder-xling/en-de/1,
https://tfhub.dev/google/universal-sentence-encoder-xling/en-es/1,
https://tfhub.dev/google/universal-sentence-encoder-xling/en-fr/1)

Thanks @akhorlin
I have had the time to test these new models. The accuracy is similar and sometimes better, however, the speed is not comparable with the original USE models. The new CMLM models are based on BERT, so it still needs to deal with tokens/encoding/word_by_word prediction and whatever happens to make the vectors to 1 per text.
The original USE models are much faster and their accuracy is still close to STOA. I would appreciate it if we can get updates from the missing TF v2 models.


feature: Builder component: ability to disable add / delete button [Forms]

Offers the possibility to disable the adding and/or removing of items (blocks) in the block builder component. Along with the existing "disableItemMovement" there will be two new methods available:
->disableItemCreation($bool | Closure $condition = true)

->disableItemDeletion($bool | Closure $condition = true)

Example
public static function form(Form $form): Form
{
    return $form
        ->schema([
            Builder::make('blocks')
                ->disableItemCreation(true)
                ->disableItemDeletion(true)

                // existing method
                ->disableItemMovement(false)
                ->blocks([
                    Builder\Block::make('heading')
                        ->schema([
                            TextInput::make('content')->required(),
                            Select::make('level')
                                ->options([
                                    'h1' => 'Heading 1',
                                    'h2' => 'Heading 2',
                                ])
                                ->required(),
                        ]),
                    Builder\Block::make('paragraph')
                        ->schema([
                            MarkdownEditor::make('content')->required(),
                        ]),
                ])
        ]);
}

Would be very nice to have!

Hey! Thanks for this.
I went a bit overboard and also implemented this for the repeater, and refreshed the UI for both components:

Hey! Thanks for this.
I went a bit overboard and also implemented this for the repeater, and refreshed the UI for both components:

Thanks @danharrin, this is much better!


Feat: Improvised cart page.

Describe the feature
Hovering over the login button in the Flipkart site leads to a dropdown menu.
This feature can be added in the project.
Add ScreenShots

Record

[X] I agree to follow this project's Code of Conduct
[X] I'm a GSSOC'24 contributor
[X] I want to work on this issue

I want to work on this project.Please assign this issue to me.under gssoc24

@arghadipmanna101
I would like to work on this issue . so please assign this task to me under the label gssoc24 and level1,2,or 3


0000{ is converted to null instead of 0 when field type has PIC S9(5) in US_ASCII file

Describe the bug
In Cloud Migration Project, Cobrix library is making null for the field when we get a value like '0000{' to the type PIC s9(5) in copybook for the US_ASCII file.
In the existing on-prem system it was making 0 and not giving null for the same value.
Could you please help us to solve this
Sample US_ASCII Data:
P0000{
T12340
P89870
CopyBook:
01 Employee.
05 EmployeeType  PIC X(1).
05 Salary                PIC S9(5).
Expected Output:
EmployeeType Salary
P                       0
T                       12340
P                       89870

@yruslan - Do you have any comment for this question ?

Looks like a good catch. Will take a look

Hi, I've tried reproducing the bug using a test. And it worked for me:
https://github.com/AbsaOSS/cobrix/blob/bugfix/459-parsing-ofnumerics/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala
What options do you pass to the reader (spark.read...)?

Sorry, I see that you have the punched sign, will chek that as well :)

I confirm the error - sign overpunching works only for EBCDIC. We will add the support to ASCII as well shortly

Thanks @yruslan

Sure, I will check and let you know

It's working @yruslan . Thank You!


[RNN] dynamic input shape support

System information

OS Platform and Distribution (e.g., Linux Ubuntu 16.04): macOS 10.15.4
TensorFlow installed from (source or binary): binary
TensorFlow version (or github SHA if from source): tf-nightly (2.3.0.dev20200604)

Command used to run the converter or code if you’re using the Python API
If possible, please share a link to Colab/Jupyter/any notebook.
https://colab.research.google.com/drive/1BtT2DkitM6PQKxOAsd-S0IU2Fwh2Reo1?usp=sharing

Failure details
I was heard about the support for unknown dimensions in TF Lite (https://github.com/tensorflow/tensorflow/issues/29590#issuecomment-580951882), then I tried to convert my model mainly followed by the instructions (https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/examples/experimental_new_converter/Keras_LSTM_fusion_Codelab.ipynb), but with replacing specific dimensions in TensorSpec() to None, the model exported to SavedModel format successfully. However, the conversion of TF Lite converter didn't work fine (errors shown in Colab).
Before this, I've also tried fixed size in TenserSpec(), then called resize_tensor_input() & allocate_tensors() on the TF Lite interpreter's side, but it didn't work either :(
Are there any proper ways for model in TF Lite to accept variable shape data?

@superlyy
I ran the code shared, please find the gist here , please let us know if this confirms the issue.

@Saduf2019
I ran this code successfully before, but I think it doesn't mention about variable size input tensor any.

Hello @superlyy i dont know your use case, i was having this same issue with images, but later i realize that i will not work with image tensor but with byte strings tensor, maybe you can update your approach to work that way. i have an implementation here https://github.com/ElPapi42/deep-deblurring-model/blob/tflite/deblurrer/model/wrapper.py, check the decorator, maybe can be helpful for you.

@superlyy
Please update as per above comment.

Hey guys, @ElPapi42, thanks for the suggestions again. I try with a subclassed Model, but it still doesn't work well with LSTM. Btw, the model is actually made for MIR-related task, the inputs are time-series of preprocessed features.

Hello again! i don't have a solution, but I noticed you don't initialize the model weights before saving to disk, can you try to forward pass a batch of sample data through the architecture before doing tf.saved_model.save()?

Absolutely. Before saving model, I forward a random data to model by using fit() in here. But it shows another error in convert.convert():
None is only supported in the 1st dimension. Tensor 'args_0' has invalid shape '[None, None, 256]'.
I'm wondering if TF Lite can not only support the dynamic shape in batch size, but also the other dimensions?

I decide to write another bare minimum example, and I can confirm this only happens with LSTM layer, when changed to GRU everything works:
import tensorflow as tf
from tensorflow.keras.layers import LSTM, Bidirectional, GRU

class SubClassedModel(tf.keras.Model):
    def __init__(self):
        super(SubClassedModel, self).__init__()
        self.model = Bidirectional(GRU(4, time_major=True, return_sequences=True), name='BILSTM_OUTPUT')

    @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, 256], dtype=tf.float32)])
    def call(self, inputs):
        return self.model(inputs)

def convert(model):
    converter = tf.lite.TFLiteConverter.from_keras_model(model)
    return converter.convert()

model = SubClassedModel()
test_input = tf.random.uniform(shape=[2, 2, 256], dtype=tf.float32)
with tf.device('/cpu:0'):
    test_output = model(test_input)

tflite = convert(model)

Dont know the reason for this error, but surely must be inspected @Saduf2019

@superlyy @ElPapi42
Please share a colab gist of the error faced for us to analyse

@Saduf2019 https://colab.research.google.com/drive/1sPKS9ZhRmCVV9JB9KmY0ywH7ZxWHFKxQ?usp=sharing
If you change the LSTM layer to a GRU, everything works on 2.3.0-dev20200609. On 2.2.0 both are broken.

@ElPapi42 @superlyy Looks like this was resolved in recent tf-nightly. Please check the gist here. thanks!


⚠ FearNoPeer has degraded performance

In a73e0e9, FearNoPeer ($FNP) experienced degraded performance:

HTTP code: 200
Response time: 4075 ms

Resolved: FearNoPeer performance has improved in b29a6ad after 1 hour, 45 minutes.


io dispatcher doesn't prefetch schedule workgroups anymore

What type of PR is this：

[x] bug
[ ] feature
[x] enhancement
[ ] others

Problem Summary ：

Remove prefetch of next to be executed num_schedule workgroups for i/o dispatcher, because the overhead of calculating which workgroup to be executed is not large, and a new ready workgroup may be delayed num_schedule times.
Fix the BE uint test by adding close logic for io dispatcher.
Yield i/o task when it spent pipeline_scan_task_yield_max_tims_spent time, default is 100ms.

run starrocks_be_unittest

run starrocks_fe_unittest

run starrocks_fe_unittest


aws-xray-daemon:3.3.2 starts logging Error level message while stopping x-ray daemon container

Issue Description:
public.ecr.aws/xray/aws-xray-daemon:3.3.2 starts logging the following "Error"  ( [Error] unable to start the server: http: Server closed ) message while stopping the daemon container.
2021-06-09T12:25:40Z [Info] Starting proxy http server on <IP_ADDRESS>:2000
2021-06-09T12:26:34Z [Info] Got shutdown signal: terminated
2021-06-09T12:26:34Z [Error] unable to start the server: http: Server closed

On the other hand, I couldn't see Error message when I tested with public.ecr.aws/xray/aws-xray-daemon:3.3.1 .
aws-xray-daemon:3.3.1 only show like the following message when I set log level to info .
2021-06-09T12:35:10Z [Info] Starting proxy http server on <IP_ADDRESS>:2000
2021-06-09T12:36:37Z [Info] Got shutdown signal: terminated

How to reproduce this issue:

Start x-ray daemon container.

  $ docker pull public.ecr.aws/xray/aws-xray-daemon:3.3.2
  $ docker run -d public.ecr.aws/xray/aws-xray-daemon:3.3.2

Check container is started successfully.

  $ docker ps

Stop x-ray container after 20-30 seconds.

  $ docker stop <container-id>

Check log from the container.

  $ docker logs <container-id>

You will see "[Error] unable to start the server: http: Server closed" in the output of the container.
I tested the above step with public.ecr.aws/xray/aws-xray-daemon:3.3.1 . However, I couldn't reproduce this issue.
Note: When I start aws-xray-daemon:3.3.1, I used the following command to set log level.
  $ docker run -d public.ecr.aws/xray/aws-xray-daemon:3.3.1 -l info

Investigation:
From the error message, I guess this issue is the side effect of the following change because the following code added "log.Errorf("unable to start the server: %v", err)" in pkg/proxy/server.go to show err variable.

Add logs for ignored errors #138

Based on the above change, I think we can ignore the message. However, the [Error] level message looks to mislead anybody who use the container. So, I think the message should not have Error level log.
Could you please consider to improve the log message?
Please let me know if you need additional information.

Hi @atakidev,
Thanks for raising this. I've opened #145 to address the log message. Please let me know if you think that is satisfactory.


react-hooks

Thanks for the PR!
To better assist you, please select the type of PR you want to create.
Click the "Preview" tab above, and click on the link for the PR type:

🐛 Bug Fix
👍 New Hook
📓 Documentation Fix

Aand what is that? =)


CI: add crate publication

As discussed, we should add crate publication triggered by git tags (starting with v) for these projects:

[ ] rust-lnpbp
[ ] rgb-node
[ ] lnp_node
[ ] rust-amplify
[ ] rust-electrum-client
[ ] bitcoin_num

Please let me know if I forgot any project

Electrum didn't is a fork which we do not maintain and can't publish. Otherwise you listed them all


Remote VPN is not working if site-to-site VPN exists

ISSUE TYPE

Bug Report

COMPONENT NAME

Virtual Router

CLOUDSTACK VERSION

4.16, 4.17, main

SUMMARY

When is site to site VPN already exists then remote VPN does not work.
There was an old issue https://github.com/apache/cloudstack/issues/3654 already but even with newer strongswan libraries this was not working as expected.
This was discussed here https://wiki.strongswan.org/issues/2497 and a possible workaround or fix could be as follows,
Modify /opt/cloud/bin/configure.py:
secret.addeq("%s : PSK "%s"" % (left, psk)) to
secret.addeq("%s %%any : PSK "%s"" % (left, psk))
This needs testing and check for regressions.
STEPS TO REPRODUCE

1. Step up site-to-site VPN between two networks
2. Also enable remote VPN

EXPECTED RESULTS

As per this issue https://github.com/apache/cloudstack/issues/3654, both site to site VPN and remote VPN should work

ACTUAL RESULTS

Only either of site to site VPN or remote VPN is working

@harikrishna-patnala
PR #5375, introduced in version <IP_ADDRESS>, removed parameter %any of VPNs client-to-site (C2S) IPSec secrets:

previous structure:
<IP> %any : PSK "<PSK>"
structure after the change:
<IP> : PSK "<PSK>"

Because of that, when a VPN site-so-site (S2S) is created in parallel to a VPN C2S in the same network, the C2S will not handle any IP (%any) anymore and, as the network is being tunneled to the other VPN, the connection will be handled by the final peer. This way, when a VPN S2S is created in parallel to a VPN C2S in the same network, it is only possible to connect to the C2S with the S2S PSK.
I created PR #6907 to handle this situation, could you take a look at it?


[BUG] Members, Payments and Dashboard pages don't load properly

Describe the bug
Multiple pages don't seem to load properly and errors are thrown on the console when they are opened.
Severity
Mild
To Reproduce
Steps to reproduce the behavior:

Open the webpage following the guide on the readme
Click on members page (This is the most obvious)
Page is stuck on the loading screen indefinitely

This error occurred because I pulled from master, so I was no longer logged in. However, my web browser still had the webpage in cache, so all the components fetched from the backend would not load.
Expected behaviour
The pages are supposed to load. For the members page, you should be able to see flat members' information.
Actual behaviour
For the members page it is stuck on loading indefinitely, for other pages, some info does not load
Environment:

OS:  Windows 10
Browser chrome

Possible solution:
logging out and logging in again should solve this problem.

Will have a look and see if I can reproduce first

Hmmm, everything still seems to working for me, wonder if it's something wrong with your local build? I can try help figure it out, maybe first try making sure you have the latest version of master and then doing a fresh build.

logging out and in again fixes this issue, closed


Checkout for html and css

Changes can be made in future if needed. If you have any questions or critiques let me know open for any changes.

lgtm


Update airplane.py

Cast np.float64 to int

Great catches here, thanks again! :)
Merging these into the develop branch, which will get automatically merged into master on the next version increment - usually roughly weekly for minor versions.

Great catches here, thanks again! :)
Merging these into the develop branch, which will get automatically merged into master on the next version increment - usually roughly weekly for minor versions.


fix: reset errors when switching between bots

Description
Need to recheck errors while switching between bots.
Task Item
Closes #5785
Screenshots

Coverage decreased (-0.03%) to 53.332% when pulling 78f3e6a0e2af7d11eb954c8cbd94b8952b4460a0 on runtime into a558e784df66410003859e6dc2012ca88162e761 on main.


[Docs]: expect.extend(yourCustomMatcherHere) example no longer compiles

Page(s)
Page: https://jestjs.io/docs/expect#expectextendmatchers
Code: https://github.com/jestjs/jest/blob/main/docs/ExpectAPI.md?plain=1#L1411
Description
After updating from version 29.4.3 to 29.5.0, my custom matcher no longer works. The example from the Jest doc no longer compiles either
Type 'MatcherFunction<[floor: unknown, ceiling: unknown]>' is not assignable to type 'RawMatcherFn<MatcherContext>'.
  The 'this' types of each signature are incompatible.
    Type 'MatcherContext' is not assignable to type 'MatcherState'.
      Property 'isNot' is optional in type 'MatcherContext' but required in type 'MatcherState'.ts(2322)
index.d.ts(314, 3): The expected type comes from this index signature.

Seems like you have a mismatch in types, can you put together a minimal reproduction which shows this?

can you put together a minimal reproduction which shows this?

This is the code from the doc linked in the 1st comment, you can copy it in a test file:
import { expect, test } from '@jest/globals';
import type { MatcherFunction } from 'expect';

const toBeWithinRange: MatcherFunction<[floor: unknown, ceiling: unknown]> =
  // `floor` and `ceiling` get types from the line above
  // it is recommended to type them as `unknown` and to validate the values
  function (actual, floor, ceiling) {
    if (typeof actual !== 'number' || typeof floor !== 'number' || typeof ceiling !== 'number') {
      throw new TypeError('These must be of type number!');
    }

    const pass = actual >= floor && actual <= ceiling;
    if (pass) {
      return {
        message: () =>
          // `this` context will have correct typings
          `expected ${this.utils.printReceived(actual)} not to be within range ${this.utils.printExpected(`${floor} - ${ceiling}`)}`,
        pass: true,
      };
    } else {
      return {
        message: () =>
          `expected ${this.utils.printReceived(actual)} to be within range ${this.utils.printExpected(`${floor} - ${ceiling}`)}`,
        pass: false,
      };
    }
  };

expect.extend({
  toBeWithinRange,
});

declare module 'expect' {
  interface AsymmetricMatchers {
    toBeWithinRange(floor: number, ceiling: number): void;
  }
  interface Matchers<R> {
    toBeWithinRange(floor: number, ceiling: number): R;
  }
}

describe('potato', () => {
  test('is within range', () => expect(100).toBeWithinRange(90, 110));

  test('is NOT within range', () => expect(101).not.toBeWithinRange(0, 100));

  test('asymmetric ranges', () => {
    expect({ apples: 6, bananas: 3 }).toEqual({
      apples: expect.toBeWithinRange(1, 10),
      bananas: expect.not.toBeWithinRange(11, 20),
    });
  });
});

That's the example from https://github.com/jestjs/jest/blob/349afc3035462d3afbfe9dfa1be43674c0092e2b/examples/expect-extend/toBeWithinRange.ts, which is tested in CI.
Also pasting that into a new repository does not reproduce.
https://github.com/SimenB/jest-extend-repro

^Hmm same for me, your repo doesn't cause the error message 🤔

I deleted my dependencies lock file, regenerate it and my problem goes away
Thanks for looking into this. @SimenB , it helped me narrow down where my problem was


[NhkRadiru] Unable to download JSON metadata: HTTP Error 404 / NHK

DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE

[X] I understand that I will be blocked if I intentionally remove or skip any mandatory* field

Checklist

[X] I'm reporting that yt-dlp is broken on a supported site
[X] I've verified that I have updated yt-dlp to nightly or master (update instructions)
[X] I've checked that all provided URLs are playable in a browser with the same IP and same login details
[X] I've checked that all URLs and arguments with special characters are properly quoted or escaped
[X] I've searched known issues and the bugtracker for similar issues including closed ones. DO NOT post duplicates
[X] I've read the guidelines for opening an issue
[ ] I've read about sharing account credentials and I'm willing to share it if required

Region
No response
Provide a description that is worded well enough to be understood
I cannot download the program www.nhk.or.jp/radio/player/ondemand.html?p=7413_01_4008048. But when I want to save a live stream such as www.nhk.or.jp/radio/player/?ch=fm or www.nhk.or.jp/radio/player/?ch=r2 it works. About a month ago I was able to download a different program https://www.nhk.or.jp/radio/ondemand/detail.html?p=8748_01 when it was still available on demand. See below for error output.
Provide verbose output that clearly demonstrates the problem

[X] Run your yt-dlp command with -vU flag added (yt-dlp -vU <your command line>)
[ ] If using API, add 'verbose': True to YoutubeDL params instead
[X] Copy the WHOLE output (starting with [debug] Command-line config) and insert it below

Complete Verbose Output
[debug] Command-line config: ['-vU', 'www.nhk.or.jp/radio/player/ondemand.html?p=7413_01_4008048']
[debug] User config "/root/yt-dlp.conf": ['--merge-output-format', 'mkv', '--embed-metadata', '--embed-thumbnail', '--parse-metadata', 'description:(?s)(?P<meta_comment>.+)', '--parse-metadata', 'uploader_id:(?s)(?P<meta_description>.+)', '--parse-metadata', ':(?P<meta_synopsis>)', '--parse-metadata', 'playlist_id:(?s)(?P<meta_playlist_id>.+)', '--parse-metadata', 'playlist_title:(?s)(?P<meta_playlist_title>.+)', '--parse-metadata', 'view_count:(?s)(?P<meta_view_count>.+)', '-o', '%(uploader)s/%(title)s [%(id)s].%(ext)s', '-S', 'res,fps,vcodec:av1', '--video-multistreams', '--audio-multistreams', '--embed-subs', '--sub-langs', 'all,-live_chat', '--cookies', 'cookies.txT']
[debug] Encodings: locale UTF-8, fs utf-8, pref UTF-8, out utf-8, error utf-8, screen utf-8
[debug] yt-dlp version<EMAIL_ADDRESS>from yt-dlp/yt-dlp [12b248ce6]
[debug] Python 3.11.9 (CPython amd64 64bit) - FreeBSD-13.1-RELEASE-p9-amd64-64bit-ELF (OpenSSL 1.1.1t-freebsd  7 Feb 2023, libc 7)
[debug] exe versions: ffmpeg 6.1.1 (setts), ffprobe 6.1.1
[debug] Optional libraries: Cryptodome-3.20.0, brotli-1.1.0, certifi-2024.02.02, mutagen-1.47.0, requests-2.32.2, sqlite3-3.45.1, urllib3-1.26.18, websockets-12.0
[debug] Proxy map: {}
[debug] Request Handlers: urllib, requests, websockets
[debug] Loaded 1820 extractors
[debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
Latest version<EMAIL_ADDRESS>from yt-dlp/yt-dlp
yt-dlp is up to date<EMAIL_ADDRESS>from yt-dlp/yt-dlp)
[generic] Extracting URL: www.nhk.or.jp/radio/player/ondemand.html?p=7413_01_4008048
WARNING: [generic] The url doesn't specify the protocol, trying with http
[debug] Using fake IP <IP_ADDRESS> (JP) as X-Forwarded-For
[NhkRadiru] Downloading API config
[NhkRadiru] Extracting URL: http://www.nhk.or.jp/radio/player/ondemand.html?p=7413_01_4008048
[NhkRadiru] 7413_01: Downloading JSON metadata
ERROR: [NhkRadiru] Unable to download JSON metadata: HTTP Error 404: Not Found (caused by <HTTPError 404: Not Found>)
  File "/usr/local/bin/yt-dlp/yt_dlp/extractor/common.py", line 734, in extract
    ie_result = self._real_extract(url)
                ^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/extractor/nhk.py", line 714, in _real_extract
    meta = self._download_json(json_url, programme_id)['main']
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/extractor/common.py", line 1133, in download_content
    res = getattr(self, download_handle.__name__)(url_or_request, video_id, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/extractor/common.py", line 1093, in download_handle
    res = self._download_webpage_handle(
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/extractor/common.py", line 954, in _download_webpage_handle
    urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/extractor/common.py", line 903, in _request_webpage
    raise ExtractorError(errmsg, cause=err)

  File "/usr/local/bin/yt-dlp/yt_dlp/extractor/common.py", line 890, in _request_webpage
    return self._downloader.urlopen(self._create_request(url_or_request, data, headers, query, extensions))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/YoutubeDL.py", line 4142, in urlopen
    return self._request_director.send(req)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/networking/common.py", line 117, in send
    response = handler.send(request)
               ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/networking/_helper.py", line 208, in wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/networking/common.py", line 337, in send
    return self._send(request)
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/local/bin/yt-dlp/yt_dlp/networking/_requests.py", line 366, in _send
    raise HTTPError(res, redirect_loop=max_redirects_exceeded)
yt_dlp.networking.exceptions.HTTPError: HTTP Error 404: Not Found

cc @garret1317

thanks for the ping bashonly
i noticed that they'd switched to a new API on the website a week or so ago, so I've actually aready got a patch mostly-ready
just didn't see the point of getting it merged when the old one still worked, perhaps that was unwise of me...
give me a bit to clean it up and i'll try get a PR in soonish

Can I download the content in another way while this is worked on?

you could clone/download the branch and run yt-dlp from there with the sh/cmd, or possibly with python -m yt_dlp in the directory(?)

Thanks. I was able to checkout your branch and download the program as well as the next episode.

you could clone/download the branch and run yt-dlp from there with the sh/cmd that's what i do for testing at least

Can confirm it worked. That said, is it just me or are the download speeds slower than before? I can't really compare since I run it from a cron job.

That said, is it just me or are the download speeds slower than before? I can't really compare since I run it from a cron job.

Seems the same as normal to me, though like you I don't have any way to compare as I don't keep logs.
FWIW the patch doesn't add any new requests, only changes one. And the fragments seem to still be served from a local Akamai presence as normal, with the same length and bitrate.
A tip, if you're not already doing it: I like to use multi-threaded download with -N, it makes it much faster. I usually set it to 10, and that gets it "fast enough" for my needs (~800K/s, which mostly saturates my connection- a 2 hour programme downloads in just under a minute).

@scineram @JLO64 the fix has been merged, so you should be able to run yt-dlp --update-to nightly and use it in the normal way now :+1: